Skip to content

Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.

Multi-instance safety: Redis cache invalidation resilience & exactly-once scheduled tasks (#731)

Krzysztof Pajak506ab7c902df
diff --git a/Directory.Packages.props b/Directory.Packages.props
index 8b8e6d5e6..a9f87a784 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -65,6 +65,7 @@
<PackageVersion Include="Verify.MSTest" Version="31.19.1" />
<PackageVersion Include="Aspire.Hosting.AppHost" Version="13.4.3" />
<PackageVersion Include="Aspire.Hosting.MongoDB" Version="13.4.3" />
+ <PackageVersion Include="Aspire.Hosting.Redis" Version="13.4.3" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.7.0" />
<PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.7.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.16.0" />
diff --git a/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj b/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj
index 550a7d6b8..aef766843 100644
--- a/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj
+++ b/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj
@@ -16,6 +16,7 @@
<ItemGroup>
<PackageReference Include="Aspire.Hosting.AppHost" />
<PackageReference Include="Aspire.Hosting.MongoDB" />
+ <PackageReference Include="Aspire.Hosting.Redis" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Aspire.Dashboard.Sdk.$(NETCoreSdkRuntimeIdentifier)" />
diff --git a/src/Aspire/Aspire.AppHost/Program.cs b/src/Aspire/Aspire.AppHost/Program.cs
index a575b90d6..efb79fa91 100644
--- a/src/Aspire/Aspire.AppHost/Program.cs
+++ b/src/Aspire/Aspire.AppHost/Program.cs
@@ -4,6 +4,9 @@ var builder = DistributedApplication.CreateBuilder(args);
var mongo = builder.AddMongoDB("mongo").WithLifetime(ContainerLifetime.Persistent);
var mongodb = mongo.AddDatabase("Mongodb");
-builder.ConfigureGrandWebProject(mongodb);
+
+var redis = builder.AddRedis("redis").WithLifetime(ContainerLifetime.Persistent);
+
+builder.ConfigureGrandWebProject(mongodb, redis);
await builder.Build().RunAsync();
diff --git a/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs b/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs
index 5f8053133..90bb42205 100644
--- a/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs
+++ b/src/Aspire/Aspire.AppHost/ProjectConfiguration.cs
@@ -1,13 +1,30 @@
-namespace Aspire.AppHost;
+namespace Aspire.AppHost;
public static class ProjectConfiguration
{
- public static void ConfigureGrandWebProject(this IDistributedApplicationBuilder builder, IResourceBuilder<MongoDBDatabaseResource> mongodb)
+ public static void ConfigureGrandWebProject(this IDistributedApplicationBuilder builder,
+ IResourceBuilder<MongoDBDatabaseResource> mongodb,
+ IResourceBuilder<RedisResource> redis)
{
+ //run two replicas of the same application (multi-pod simulation) sharing one database
+ //and synchronizing their memory cache through the Redis pub/sub message bus.
+ //replicas (not two separate AddProject resources) so Aspire builds the project once and
+ //launches N processes - two independent builds would race over the same bin/ output and
+ //crash a process with host error 0x80008081. Aspire assigns each replica its own port,
+ //so no fixed WithHttpEndpoint here.
builder
.AddProject<Projects.Grand_Web>("grand-web")
- .WithHttpEndpoint(80, name: "front")
+ .WithReplicas(2)
.WithReference(mongodb)
- .WaitFor(mongodb);
+ .WaitFor(mongodb)
+ .WaitFor(redis)
+ //replicas share one content root - shadow copy would make them fight over the same
+ //Plugins/bin folder (files locked by the other replica)
+ .WithEnvironment("Extensions__PluginShadowCopy", "false")
+ .WithEnvironment("Redis__RedisPubSubEnabled", "true")
+ //show publish/receive debug logs of the cache message bus in the dashboard
+ .WithEnvironment("Logging__LogLevel__Grand.Infrastructure.Caching.Redis", "Debug")
+ .WithEnvironment("Redis__RedisPubSubChannel", "grandnode-cache")
+ .WithEnvironment("Redis__RedisPubSubConnectionString", redis.Resource.ConnectionStringExpression);
}
-}
\ No newline at end of file
+}
diff --git a/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs b/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs
index 89f7d02d9..8c78591b9 100644
--- a/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs
+++ b/src/Business/Grand.Business.Core/Interfaces/System/ScheduleTasks/IScheduleTaskService.cs
@@ -36,6 +36,18 @@ public interface IScheduleTaskService
/// <param name="task">Task</param>
Task UpdateTask(ScheduleTask task);
+ /// <summary>
+ /// Atomically claims a single run of the task, so that only one application instance
+ /// executes it. The claim succeeds only if LastStartUtc still has the value the caller
+ /// read (compare-and-set) - a concurrent claim by another instance makes it fail.
+ /// </summary>
+ /// <param name="taskId">Task identifier</param>
+ /// <param name="expectedLastStartUtc">LastStartUtc value read before the claim</param>
+ /// <param name="runStartUtc">New LastStartUtc value (start of the claimed run)</param>
+ /// <param name="instanceId">Unique identifier of the claiming application instance</param>
+ /// <returns>true if this instance won the claim and should execute the task</returns>
+ Task<bool> TryClaimTaskRun(string taskId, DateTime? expectedLastStartUtc, DateTime runStartUtc, string instanceId);
+
/// <summary>
/// Delete the task
/// </summary>
diff --git a/src/Core/Grand.Domain/Tasks/ScheduleTask.cs b/src/Core/Grand.Domain/Tasks/ScheduleTask.cs
index d19979140..6d773f6f6 100644
--- a/src/Core/Grand.Domain/Tasks/ScheduleTask.cs
+++ b/src/Core/Grand.Domain/Tasks/ScheduleTask.cs
@@ -10,6 +10,7 @@ public class ScheduleTask : BaseEntity
public DateTime? LastSuccessUtc { get; set; }
public string LeasedByMachineName { get; set; }
public DateTime? LeasedUntilUtc { get; set; }
+ public string LeasedByInstance { get; set; }
public int TimeInterval { get; set; }
public string StoreId { get; set; }
}
\ No newline at end of file
diff --git a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs
index 7117afb2c..4e7a613c9 100644
--- a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs
+++ b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageBus.cs
@@ -1,66 +1,85 @@
-using Grand.Infrastructure.Caching.Message;
+using Grand.Infrastructure.Caching.Message;
using Grand.Infrastructure.Configuration;
using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
using StackExchange.Redis;
-using System.Diagnostics;
using System.Text.Json;
namespace Grand.Infrastructure.Caching.Redis;
-public sealed class RedisMessageBus : IMessageBus
+public sealed class RedisMessageBus : IMessageBus, IHostedService, IDisposable
{
private static readonly string ClientId = Guid.NewGuid().ToString("N");
+
+ private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromSeconds(1);
+ private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(30);
+
+ private readonly IConnectionMultiplexer _connection;
private readonly RedisConfig _redisConfig;
private readonly IServiceProvider _serviceProvider;
private readonly ISubscriber _subscriber;
+ private readonly ILogger<RedisMessageBus> _logger;
+ private readonly CancellationTokenSource _cts = new();
+ private Task _subscribeLoop;
- public RedisMessageBus(ISubscriber subscriber, IServiceProvider serviceProvider, RedisConfig redisConfig)
+ public RedisMessageBus(IConnectionMultiplexer connection, IServiceProvider serviceProvider,
+ RedisConfig redisConfig, ILogger<RedisMessageBus> logger)
{
- _subscriber = subscriber;
+ _connection = connection;
+ _subscriber = connection.GetSubscriber();
_serviceProvider = serviceProvider;
_redisConfig = redisConfig;
- SubscribeAsync();
+ _logger = logger;
+ }
+
+ public Task StartAsync(CancellationToken cancellationToken)
+ {
+ _connection.ConnectionFailed += OnConnectionFailed;
+ _connection.ConnectionRestored += OnConnectionRestored;
+
+ //keep the subscription attempt off the startup path since Redis may be unreachable,
+ //and track the loop so it can be cancelled and awaited on shutdown
+ _subscribeLoop = SubscribeWithRetryAsync(_cts.Token);
+ return Task.CompletedTask;
+ }
+
+ public async Task StopAsync(CancellationToken cancellationToken)
+ {
+ _connection.ConnectionFailed -= OnConnectionFailed;
+ _connection.ConnectionRestored -= OnConnectionRestored;
+ await _cts.CancelAsync();
+ if (_subscribeLoop != null)
+ await _subscribeLoop.WaitAsync(cancellationToken);
}
public async Task PublishAsync<TMessage>(TMessage msg) where TMessage : IMessageEvent
{
+ var message = JsonSerializer.Serialize(new MessageEventClient {
+ ClientId = ClientId,
+ Key = msg.Key,
+ MessageType = msg.MessageType
+ });
try
{
- var client = new MessageEventClient {
- ClientId = ClientId,
- Key = msg.Key,
- MessageType = msg.MessageType
- };
- var message = JsonSerializer.Serialize(client);
- await _subscriber.PublishAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), message);
+ var receivers =
+ await _subscriber.PublishAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), message);
+ if (_logger.IsEnabled(LogLevel.Debug))
+ _logger.LogDebug(
+ "Published cache invalidation message (type: {MessageType}, key: {Key}), delivered to {Receivers} subscriber(s)",
+ (MessageEventType)msg.MessageType, msg.Key, receivers);
}
catch (Exception ex)
{
- Debug.WriteLine(ex.Message);
+ _logger.LogError(ex,
+ "Failed to publish cache invalidation message (type: {MessageType}, key: {Key}) - other instances may serve stale data until cache expiration",
+ (MessageEventType)msg.MessageType, msg.Key);
}
}
public Task SubscribeAsync()
{
- _ = _subscriber.SubscribeAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), (_, redisValue) =>
- {
- try
- {
- MessageEventClient message = null;
- if (!redisValue.IsNull)
- {
- // Use the string overload explicitly to resolve ambiguity
- message = JsonSerializer.Deserialize<MessageEventClient>(redisValue.ToString());
- }
- if (message != null && message.ClientId != ClientId)
- OnSubscriptionChanged(message);
- }
- catch (Exception ex)
- {
- Debug.WriteLine(ex.Message);
- }
- });
- return Task.CompletedTask;
+ return _subscriber.SubscribeAsync(RedisChannel.Literal(_redisConfig.RedisPubSubChannel), OnMessage);
}
public void OnSubscriptionChanged(IMessageEvent message)
@@ -80,4 +99,93 @@ public sealed class RedisMessageBus : IMessageBus
break;
}
}
-}
\ No newline at end of file
+
+ public void Dispose()
+ {
+ //event handlers are normally detached in StopAsync; detach again in case the
+ //hosted-service lifecycle was skipped, and release the cancellation source
+ _connection.ConnectionFailed -= OnConnectionFailed;
+ _connection.ConnectionRestored -= OnConnectionRestored;
+ if (!_cts.IsCancellationRequested)
+ _cts.Cancel();
+ _cts.Dispose();
+ }
+
+ private async Task SubscribeWithRetryAsync(CancellationToken cancellationToken)
+ {
+ var delay = InitialRetryDelay;
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await SubscribeAsync();
+ if (_logger.IsEnabled(LogLevel.Information))
+ _logger.LogInformation("Subscribed to Redis pub/sub channel {Channel}",
+ _redisConfig.RedisPubSubChannel);
+ return;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex,
+ "Failed to subscribe to Redis pub/sub channel {Channel}, retrying in {Delay}s",
+ _redisConfig.RedisPubSubChannel, delay.TotalSeconds);
+ try
+ {
+ await Task.Delay(delay, cancellationToken);
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ delay = delay * 2 > MaxRetryDelay ? MaxRetryDelay : delay * 2;
+ }
+ }
+ }
+
+ private void OnMessage(RedisChannel channel, RedisValue redisValue)
+ {
+ try
+ {
+ if (redisValue.IsNull) return;
+ var message = JsonSerializer.Deserialize<MessageEventClient>(redisValue.ToString());
+ if (message != null && message.ClientId != ClientId)
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ _logger.LogDebug(
+ "Received cache invalidation message (type: {MessageType}, key: {Key}) from client {ClientId}",
+ (MessageEventType)message.MessageType, message.Key, message.ClientId);
+ OnSubscriptionChanged(message);
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to process cache invalidation message from Redis");
+ }
+ }
+
+ private void OnConnectionFailed(object sender, ConnectionFailedEventArgs e)
+ {
+ _logger.LogWarning(e.Exception,
+ "Redis connection failed ({ConnectionType}, {FailureType}) - cache invalidation messages may be lost while disconnected",
+ e.ConnectionType, e.FailureType);
+ }
+
+ private void OnConnectionRestored(object sender, ConnectionFailedEventArgs e)
+ {
+ //invalidation messages published while this instance was disconnected are lost
+ //(Redis pub/sub has no replay), so the local cache can no longer be trusted
+ if (e.ConnectionType != ConnectionType.Subscription) return;
+ _logger.LogWarning(
+ "Redis subscription connection restored - clearing local cache to drop entries that may have missed invalidation");
+ try
+ {
+ using var scope = _serviceProvider.CreateScope();
+ var cache = scope.ServiceProvider.GetRequiredService<ICacheBase>();
+ _ = cache.Clear(false);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to clear local cache after Redis connection was restored");
+ }
+ }
+}
diff --git a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs
index 6fc492267..954162361 100644
--- a/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs
+++ b/src/Core/Grand.Infrastructure/Caching/Redis/RedisMessageCacheManager.cs
@@ -1,4 +1,4 @@
-using Grand.Infrastructure.Caching.Message;
+using Grand.Infrastructure.Caching.Message;
using Grand.Infrastructure.Configuration;
using MediatR;
using Microsoft.Extensions.Caching.Memory;
@@ -7,13 +7,11 @@ namespace Grand.Infrastructure.Caching.Redis;
public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase
{
- private readonly IMemoryCache _cache;
private readonly IMessageBus _messageBus;
public RedisMessageCacheManager(IMemoryCache cache, IMediator mediator, IMessageBus messageBus, CacheConfig config)
: base(cache, mediator, config)
{
- _cache = cache;
_messageBus = messageBus;
}
@@ -22,14 +20,13 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase
/// </summary>
/// <param name="key">Key of cached item</param>
/// <param name="publisher">Publisher</param>
- public override Task RemoveAsync(string key, bool publisher = true)
+ public override async Task RemoveAsync(string key, bool publisher = true)
{
- _cache.Remove(key);
+ await base.RemoveAsync(key, false);
if (publisher)
- _messageBus.PublishAsync(new MessageEvent { Key = key, MessageType = (int)MessageEventType.RemoveKey });
-
- return Task.CompletedTask;
+ await _messageBus.PublishAsync(new MessageEvent
+ { Key = key, MessageType = (int)MessageEventType.RemoveKey });
}
/// <summary>
@@ -37,16 +34,13 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase
/// </summary>
/// <param name="prefix">String prefix</param>
/// <param name="publisher">publisher</param>
- public override Task RemoveByPrefix(string prefix, bool publisher = true)
+ public override async Task RemoveByPrefix(string prefix, bool publisher = true)
{
- var entriesToRemove = CacheEntries.Where(x => x.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase));
- foreach (var cacheEntries in entriesToRemove) _cache.Remove(cacheEntries.Key);
+ await base.RemoveByPrefix(prefix, false);
if (publisher)
- _messageBus.PublishAsync(new MessageEvent
+ await _messageBus.PublishAsync(new MessageEvent
{ Key = prefix, MessageType = (int)MessageEventType.RemoveByPrefix });
-
- return Task.CompletedTask;
}
/// <summary>
@@ -56,8 +50,9 @@ public class RedisMessageCacheManager : MemoryCacheBase, ICacheBase
public override async Task Clear(bool publisher = true)
{
await base.Clear(publisher);
- if (publisher)
- await _messageBus.PublishAsync(new MessageEvent { Key = "", MessageType = (int)MessageEventType.ClearCache });
+ if (publisher)
+ await _messageBus.PublishAsync(new MessageEvent
+ { Key = "", MessageType = (int)MessageEventType.ClearCache });
}
-}
\ No newline at end of file
+}
diff --git a/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs b/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs
index c17036276..bc4670bb4 100644
--- a/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs
+++ b/src/Modules/Grand.Module.ScheduledTasks/BackgroundServices/ScheduleTaskService.cs
@@ -78,6 +78,29 @@ public class ScheduleTaskService : IScheduleTaskService
await _taskRepository.UpdateAsync(task);
}
+ /// <summary>
+ /// Atomically claims a single run of the task (compare-and-set on LastStartUtc)
+ /// </summary>
+ public virtual async Task<bool> TryClaimTaskRun(string taskId, DateTime? expectedLastStartUtc,
+ DateTime runStartUtc, string instanceId)
+ {
+ //truncate to milliseconds so the value survives the database round-trip unchanged
+ runStartUtc = new DateTime(runStartUtc.Ticks - runStartUtc.Ticks % TimeSpan.TicksPerMillisecond,
+ DateTimeKind.Utc);
+
+ //conditional update - matches only when no other instance has claimed the run
+ //in the meantime; on MongoDB a single update is atomic, so exactly one instance wins
+ await _taskRepository.UpdateOneAsync(
+ x => x.Id == taskId && x.LastStartUtc == expectedLastStartUtc,
+ UpdateBuilder<ScheduleTask>.Create()
+ .Set(x => x.LastStartUtc, runStartUtc)
+ .Set(x => x.LeasedByInstance, instanceId));
+
+ //UpdateOneAsync does not report whether the filter matched - read back the winner
+ var task = await GetTaskById(taskId);
+ return task != null && task.LeasedByInstance == instanceId && task.LastStartUtc == runStartUtc;
+ }
+
/// <summary>
/// Delete the task
/// </summary>
diff --git a/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.cs b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.cs
new file mode 100644
index 000000000..656c4fcbb
--- /dev/null
+++ b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageBusTests.cs
@@ -0,0 +1,169 @@
+using Grand.Infrastructure.Caching;
+using Grand.Infrastructure.Caching.Message;
+using Grand.Infrastructure.Caching.Redis;
+using Grand.Infrastructure.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using StackExchange.Redis;
+using System.Net;
+using System.Text.Json;
+
+namespace Grand.Infrastructure.Tests.Caching.Redis;
+
+[TestClass]
+public class RedisMessageBusTests
+{
+ private Mock<IConnectionMultiplexer> _connectionMock;
+ private Mock<ISubscriber> _subscriberMock;
+ private Mock<ICacheBase> _cacheMock;
+ private RedisConfig _config;
+ private RedisMessageBus _bus;
+
+ [TestInitialize]
+ public void Init()
+ {
+ _subscriberMock = new Mock<ISubscriber>();
+ _subscriberMock.Setup(s => s.SubscribeAsync(It.IsAny<RedisChannel>(),
+ It.IsAny<Action<RedisChannel, RedisValue>>(), It.IsAny<CommandFlags>()))
+ .Returns(Task.CompletedTask);
+
+ _connectionMock = new Mock<IConnectionMultiplexer>();
+ _connectionMock.Setup(c => c.GetSubscriber(It.IsAny<object>())).Returns(_subscriberMock.Object);
+
+ _cacheMock = new Mock<ICacheBase>();
+ _cacheMock.Setup(c => c.RemoveAsync(It.IsAny<string>(), It.IsAny<bool>())).Returns(Task.CompletedTask);
+ _cacheMock.Setup(c => c.RemoveByPrefix(It.IsAny<string>(), It.IsAny<bool>())).Returns(Task.CompletedTask);
+ _cacheMock.Setup(c => c.Clear(It.IsAny<bool>())).Returns(Task.CompletedTask);
+
+ var services = new ServiceCollection();
+ services.AddSingleton(_cacheMock.Object);
+ var serviceProvider = services.BuildServiceProvider();
+
+ _config = new RedisConfig { RedisPubSubChannel = "test-channel" };
+ _bus = new RedisMessageBus(_connectionMock.Object, serviceProvider, _config,
+ new Mock<ILogger<RedisMessageBus>>().Object);
+ }
+
+ [TestCleanup]
+ public void Cleanup()
+ {
+ _bus.Dispose();
+ }
+
+ [TestMethod]
+ public async Task StartAsync_SubscribesToConfiguredChannel()
+ {
+ await _bus.StartAsync(CancellationToken.None);
+
+ //subscription runs on a tracked background loop - wait for it
+ for (var i = 0; i < 100; i++)
+ {
+ try
+ {
+ _subscriberMock.Verify(s => s.SubscribeAsync(
+ It.Is<RedisChannel>(c => c == RedisChannel.Literal("test-channel")),
+ It.IsAny<Action<RedisChannel, RedisValue>>(), It.IsAny<CommandFlags>()), Times.AtLeastOnce);
+ return;
+ }
+ catch (MockException)
+ {
+ await Task.Delay(20);
+ }
+ }
+
+ Assert.Fail("SubscribeAsync was not called on the configured channel");
+ }
+
+ [TestMethod]
+ public async Task StopAsync_CompletesAndStopsSubscribing()
+ {
+ await _bus.StartAsync(CancellationToken.None);
+ await _bus.StopAsync(CancellationToken.None);
+ }
+
+ [TestMethod]
+ public async Task PublishAsync_SendsSerializedMessageToConfiguredChannel()
+ {
+ RedisValue publishedValue = default;
+ _subscriberMock.Setup(s =>
+ s.PublishAsync(It.IsAny<RedisChannel>(), It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()))
+ .Callback<RedisChannel, RedisValue, CommandFlags>((_, value, _) => publishedValue = value)
+ .ReturnsAsync(1);
+
+ await _bus.PublishAsync(new MessageEvent { Key = "key", MessageType = (int)MessageEventType.RemoveKey });
+
+ _subscriberMock.Verify(s => s.PublishAsync(
+ It.Is<RedisChannel>(c => c == RedisChannel.Literal("test-channel")),
+ It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()), Times.Once);
+
+ var message = JsonSerializer.Deserialize<MessageEventClient>(publishedValue.ToString());
+ Assert.IsNotNull(message);
+ Assert.AreEqual("key", message.Key);
+ Assert.AreEqual((int)MessageEventType.RemoveKey, message.MessageType);
+ Assert.IsFalse(string.IsNullOrEmpty(message.ClientId));
+ }
+
+ [TestMethod]
+ public async Task PublishAsync_SubscriberThrows_DoesNotPropagateException()
+ {
+ _subscriberMock.Setup(s =>
+ s.PublishAsync(It.IsAny<RedisChannel>(), It.IsAny<RedisValue>(), It.IsAny<CommandFlags>()))
+ .ThrowsAsync(new RedisConnectionException(ConnectionFailureType.SocketFailure, "connection lost"));
+
+ await _bus.PublishAsync(new MessageEvent { Key = "key", MessageType = (int)MessageEventType.RemoveKey });
+ }
+
+ [TestMethod]
+ public void OnSubscriptionChanged_RemoveKey_RemovesFromLocalCacheWithoutRepublishing()
+ {
+ _bus.OnSubscriptionChanged(new MessageEventClient
+ { ClientId = "other", Key = "key", MessageType = (int)MessageEventType.RemoveKey });
+
+ _cacheMock.Verify(c => c.RemoveAsync("key", false), Times.Once);
+ }
+
+ [TestMethod]
+ public void OnSubscriptionChanged_RemoveByPrefix_RemovesFromLocalCacheWithoutRepublishing()
+ {
+ _bus.OnSubscriptionChanged(new MessageEventClient
+ { ClientId = "other", Key = "prefix", MessageType = (int)MessageEventType.RemoveByPrefix });
+
+ _cacheMock.Verify(c => c.RemoveByPrefix("prefix", false), Times.Once);
+ }
+
+ [TestMethod]
+ public void OnSubscriptionChanged_ClearCache_ClearsLocalCacheWithoutRepublishing()
+ {
+ _bus.OnSubscriptionChanged(new MessageEventClient
+ { ClientId = "other", Key = "", MessageType = (int)MessageEventType.ClearCache });
+
+ _cacheMock.Verify(c => c.Clear(false), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task ConnectionRestored_SubscriptionConnection_ClearsLocalCache()
+ {
+ await _bus.StartAsync(CancellationToken.None);
+
+ //messages published while disconnected are lost - the local cache must be dropped
+ _connectionMock.Raise(c => c.ConnectionRestored += null,
+ new ConnectionFailedEventArgs(_connectionMock.Object, new DnsEndPoint("localhost", 6379),
+ ConnectionType.Subscription, ConnectionFailureType.SocketClosed, null, "test"));
+
+ _cacheMock.Verify(c => c.Clear(false), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task ConnectionRestored_InteractiveConnection_DoesNotClearLocalCache()
+ {
+ await _bus.StartAsync(CancellationToken.None);
+
+ _connectionMock.Raise(c => c.ConnectionRestored += null,
+ new ConnectionFailedEventArgs(_connectionMock.Object, new DnsEndPoint("localhost", 6379),
+ ConnectionType.Interactive, ConnectionFailureType.SocketClosed, null, "test"));
+
+ _cacheMock.Verify(c => c.Clear(It.IsAny<bool>()), Times.Never);
+ }
+}
diff --git a/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.cs b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.cs
new file mode 100644
index 000000000..cc2c7b064
--- /dev/null
+++ b/src/Tests/Grand.Infrastructure.Tests/Caching/Redis/RedisMessageCacheManagerTests.cs
@@ -0,0 +1,102 @@
+using Grand.Infrastructure.Caching.Message;
+using Grand.Infrastructure.Caching.Redis;
+using Grand.Infrastructure.Configuration;
+using MediatR;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+namespace Grand.Infrastructure.Tests.Caching.Redis;
+
+[TestClass]
+public class RedisMessageCacheManagerTests
+{
+ private Mock<IMessageBus> _messageBusMock;
+ private IMemoryCache _memoryCache;
+ private RedisMessageCacheManager _service;
+
+ [TestInitialize]
+ public void Init()
+ {
+ var services = new ServiceCollection();
+ services.AddMemoryCache();
+ _memoryCache = services.BuildServiceProvider().GetService<IMemoryCache>();
+ _messageBusMock = new Mock<IMessageBus>();
+ _service = new RedisMessageCacheManager(_memoryCache, new Mock<IMediator>().Object,
+ _messageBusMock.Object, new CacheConfig { DefaultCacheTimeMinutes = 1 });
+ }
+
+ [TestMethod]
+ public async Task RemoveAsync_RemovesKeyAndPublishesMessage()
+ {
+ _service.Get("key", () => "value");
+
+ await _service.RemoveAsync("key");
+
+ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>
+ e.Key == "key" && e.MessageType == (int)MessageEventType.RemoveKey)), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task RemoveAsync_PublisherDisabled_DoesNotPublish()
+ {
+ _service.Get("key", () => "value");
+
+ await _service.RemoveAsync("key", false);
+
+ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);
+ }
+
+ [TestMethod]
+ public async Task RemoveByPrefix_RemovesMatchingKeysAndPublishesMessage()
+ {
+ _service.Get("prefix-1", () => "value");
+ _service.Get("prefix-2", () => "value");
+ _service.Get("other", () => "value");
+
+ await _service.RemoveByPrefix("prefix");
+
+ Assert.IsFalse(_memoryCache.TryGetValue("prefix-1", out _));
+ Assert.IsFalse(_memoryCache.TryGetValue("prefix-2", out _));
+ Assert.IsTrue(_memoryCache.TryGetValue("other", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>
+ e.Key == "prefix" && e.MessageType == (int)MessageEventType.RemoveByPrefix)), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task RemoveByPrefix_PublisherDisabled_DoesNotPublish()
+ {
+ _service.Get("prefix-1", () => "value");
+
+ await _service.RemoveByPrefix("prefix", false);
+
+ Assert.IsFalse(_memoryCache.TryGetValue("prefix-1", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);
+ }
+
+ [TestMethod]
+ public async Task Clear_PublishesClearCacheMessage()
+ {
+ _service.Get("key", () => "value");
+
+ await _service.Clear();
+
+ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.Is<MessageEvent>(e =>
+ e.MessageType == (int)MessageEventType.ClearCache)), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task Clear_PublisherDisabled_DoesNotPublish()
+ {
+ _service.Get("key", () => "value");
+
+ await _service.Clear(false);
+
+ Assert.IsFalse(_memoryCache.TryGetValue("key", out _));
+ _messageBusMock.Verify(m => m.PublishAsync(It.IsAny<MessageEvent>()), Times.Never);
+ }
+}
diff --git a/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj b/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj
index aa8e24bc4..dc6e1d9a2 100644
--- a/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj
+++ b/src/Tests/Grand.Modules.Tests/Grand.Modules.Tests.csproj
@@ -21,6 +21,7 @@
<ProjectReference Include="..\..\Business\Grand.Business.Messages\Grand.Business.Messages.csproj" />
<ProjectReference Include="..\..\Modules\Grand.Module.Migration\Grand.Module.Migration.csproj" />
<ProjectReference Include="..\..\Modules\Grand.Module.ScheduledTasks\Grand.Module.ScheduledTasks.csproj" />
+ <ProjectReference Include="..\Grand.Data.Tests\Grand.Data.Tests.csproj" />
</ItemGroup>
</Project>
diff --git a/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.cs b/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.cs
new file mode 100644
index 000000000..7663112f3
--- /dev/null
+++ b/src/Tests/Grand.Modules.Tests/Services/BackgroundService/ScheduleTaskServiceTests.cs
@@ -0,0 +1,83 @@
+using Grand.Data.Tests.MongoDb;
+using Grand.Domain.Tasks;
+using Grand.Module.ScheduledTasks.BackgroundServices;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Grand.Modules.Tests.Services.BackgroundService;
+
+[TestClass]
+public class ScheduleTaskServiceTests
+{
+ private MongoDBRepositoryTest<ScheduleTask> _repository;
+ private ScheduleTaskService _service;
+
+ [TestInitialize]
+ public void Init()
+ {
+ _repository = new MongoDBRepositoryTest<ScheduleTask>();
+ _service = new ScheduleTaskService(_repository);
+ }
+
+ private async Task<ScheduleTask> InsertTask(DateTime? lastStartUtc = null)
+ {
+ var task = new ScheduleTask {
+ ScheduleTaskName = "TestTask",
+ Enabled = true,
+ TimeInterval = 1,
+ LastStartUtc = lastStartUtc
+ };
+ await _service.InsertTask(task);
+ //read back so LastStartUtc has the value as stored in the database (ms precision)
+ return await _service.GetTaskById(task.Id);
+ }
+
+ [TestMethod]
+ public async Task TryClaimTaskRun_FirstClaim_ReturnsTrueAndSetsLease()
+ {
+ var task = await InsertTask();
+ var runStartUtc = DateTime.UtcNow;
+
+ var claimed = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, runStartUtc, "instance-a");
+
+ Assert.IsTrue(claimed);
+ var updated = await _service.GetTaskById(task.Id);
+ Assert.AreEqual("instance-a", updated.LeasedByInstance);
+ Assert.IsNotNull(updated.LastStartUtc);
+ }
+
+ [TestMethod]
+ public async Task TryClaimTaskRun_StaleExpectedValue_ReturnsFalseAndKeepsWinner()
+ {
+ var task = await InsertTask(DateTime.UtcNow.AddMinutes(-10));
+
+ //instance A claims the run
+ var claimedByA = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-a");
+ //instance B still holds the old LastStartUtc value - its claim must fail
+ var claimedByB = await _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-b");
+
+ Assert.IsTrue(claimedByA);
+ Assert.IsFalse(claimedByB);
+ var updated = await _service.GetTaskById(task.Id);
+ Assert.AreEqual("instance-a", updated.LeasedByInstance);
+ }
+
+ [TestMethod]
+ public async Task TryClaimTaskRun_ConcurrentClaims_ExactlyOneWins()
+ {
+ var task = await InsertTask(DateTime.UtcNow.AddMinutes(-10));
+
+ var claims = await Task.WhenAll(
+ _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-1"),
+ _service.TryClaimTaskRun(task.Id, task.LastStartUtc, DateTime.UtcNow, "instance-2"));
+
+ Assert.AreEqual(1, claims.Count(x => x));
+ }
+
+ [TestMethod]
+ public async Task TryClaimTaskRun_NonExistingTask_ReturnsFalse()
+ {
+ var claimed = await _service.TryClaimTaskRun("000000000000000000000000", null, DateTime.UtcNow, "instance-a");
+
+ Assert.IsFalse(claimed);
+ }
+}
diff --git a/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs
new file mode 100644
index 000000000..eba0394a8
--- /dev/null
+++ b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs
@@ -0,0 +1,145 @@
+using Grand.Business.Core.Interfaces.System.ScheduleTasks;
+using Grand.Domain.Tasks;
+using Grand.Infrastructure;
+using Grand.Web.Common.Infrastructure;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+namespace Grand.Web.Common.Tests.Infrastructure;
+
+[TestClass]
+public class BackgroundServiceTaskTests
+{
+ private const string TaskName = "TestTask";
+
+ private Mock<IScheduleTaskService> _scheduleTaskServiceMock;
+ private Mock<IScheduleTask> _scheduleTaskMock;
+ private ScheduleTask _task;
+ private IServiceProvider _serviceProvider;
+
+ [TestInitialize]
+ public void Init()
+ {
+ _task = new ScheduleTask {
+ Id = "1",
+ ScheduleTaskName = TaskName,
+ Enabled = true,
+ TimeInterval = 60
+ };
+
+ _scheduleTaskServiceMock = new Mock<IScheduleTaskService>();
+ _scheduleTaskServiceMock.Setup(s => s.GetTaskByName(TaskName)).ReturnsAsync(_task);
+ _scheduleTaskServiceMock.Setup(s => s.UpdateTask(It.IsAny<ScheduleTask>()))
+ .Returns(Task.CompletedTask);
+
+ _scheduleTaskMock = new Mock<IScheduleTask>();
+ _scheduleTaskMock.Setup(t => t.Execute()).Returns(Task.CompletedTask);
+
+ var storeContextSetter = new Mock<IStoreContextSetter>();
+ storeContextSetter.Setup(s => s.InitializeStoreContext(It.IsAny<string>()))
+ .ReturnsAsync(Mock.Of<IStoreContext>());
+ var workContextSetter = new Mock<IWorkContextSetter>();
+ workContextSetter.Setup(s => s.InitializeWorkContext(It.IsAny<string>()))
+ .ReturnsAsync(Mock.Of<IWorkContext>());
+
+ var services = new ServiceCollection();
+ services.AddLogging();
+ services.AddSingleton(_scheduleTaskServiceMock.Object);
+ services.AddKeyedSingleton(TaskName, _scheduleTaskMock.Object);
+ services.AddSingleton(Mock.Of<IContextAccessor>());
+ services.AddSingleton(storeContextSetter.Object);
+ services.AddSingleton(workContextSetter.Object);
+ _serviceProvider = services.BuildServiceProvider();
+ }
+
+ [TestMethod]
+ public async Task ExecuteAsync_ClaimWon_ExecutesTaskAndPersistsResult()
+ {
+ _scheduleTaskServiceMock.Setup(s => s.TryClaimTaskRun(_task.Id, It.IsAny<DateTime?>(),
+ It.IsAny<DateTime>(), It.IsAny<string>()))
+ .ReturnsAsync(true);
+
+ var service = new BackgroundServiceTask(TaskName, _serviceProvider);
+ using var cts = new CancellationTokenSource();
+ await service.StartAsync(cts.Token);
+
+ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>
+ i.Method.Name == nameof(IScheduleTaskService.UpdateTask)));
+ cts.Cancel();
+
+ _scheduleTaskMock.Verify(t => t.Execute(), Times.Once);
+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.Is<ScheduleTask>(x =>
+ x.LastSuccessUtc != null && x.LastStartUtc != null)), Times.Once);
+ }
+
+ [TestMethod]
+ public async Task ExecuteAsync_ClaimLost_DoesNotExecuteAndDoesNotPersist()
+ {
+ _scheduleTaskServiceMock.Setup(s => s.TryClaimTaskRun(_task.Id, It.IsAny<DateTime?>(),
+ It.IsAny<DateTime>(), It.IsAny<string>()))
+ .ReturnsAsync(false);
+
+ var service = new BackgroundServiceTask(TaskName, _serviceProvider);
+ using var cts = new CancellationTokenSource();
+ await service.StartAsync(cts.Token);
+
+ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>
+ i.Method.Name == nameof(IScheduleTaskService.TryClaimTaskRun)));
+ //give the loop a moment to (incorrectly) execute the task if the guard is broken
+ await Task.Delay(200);
+ cts.Cancel();
+
+ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);
+ //losing instance must not overwrite the winner's data with a stale entity
+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.IsAny<ScheduleTask>()), Times.Never);
+ }
+
+ [TestMethod]
+ public async Task ExecuteAsync_TaskNotDueYet_DoesNotClaimNorExecute()
+ {
+ _task.LastStartUtc = DateTime.UtcNow;
+
+ var service = new BackgroundServiceTask(TaskName, _serviceProvider);
+ using var cts = new CancellationTokenSource();
+ await service.StartAsync(cts.Token);
+
+ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>
+ i.Method.Name == nameof(IScheduleTaskService.GetTaskByName)));
+ await Task.Delay(200);
+ cts.Cancel();
+
+ _scheduleTaskServiceMock.Verify(s => s.TryClaimTaskRun(It.IsAny<string>(), It.IsAny<DateTime?>(),
+ It.IsAny<DateTime>(), It.IsAny<string>()), Times.Never);
+ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);
+ _scheduleTaskServiceMock.Verify(s => s.UpdateTask(It.IsAny<ScheduleTask>()), Times.Never);
+ }
+
+ [TestMethod]
+ public async Task ExecuteAsync_TaskLeasedByOtherMachine_DoesNotExecute()
+ {
+ _task.LeasedByMachineName = "other-machine";
+
+ var service = new BackgroundServiceTask(TaskName, _serviceProvider);
+ using var cts = new CancellationTokenSource();
+ await service.StartAsync(cts.Token);
+
+ await WaitFor(() => _scheduleTaskServiceMock.Invocations.Any(i =>
+ i.Method.Name == nameof(IScheduleTaskService.GetTaskByName)));
+ await Task.Delay(200);
+ cts.Cancel();
+
+ _scheduleTaskMock.Verify(t => t.Execute(), Times.Never);
+ }
+
+ private static async Task WaitFor(Func<bool> condition)
+ {
+ for (var i = 0; i < 300; i++)
+ {
+ if (condition()) return;
+ await Task.Delay(10);
+ }
+
+ Assert.Fail("Condition was not met within the timeout");
+ }
+}
diff --git a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
index dcc0e885e..cc03ed0d6 100644
--- a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
+++ b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
@@ -9,6 +9,10 @@ namespace Grand.Web.Common.Infrastructure;
public class BackgroundServiceTask : BackgroundService
{
+ //unique per process - pod/machine names are not stable across scaling, so a random
+ //id generated at startup is all that is needed to tell instances apart
+ private static readonly string InstanceId = Guid.NewGuid().ToString("N");
+
private readonly IServiceProvider _serviceProvider;
private readonly string Name;
@@ -40,6 +44,7 @@ public class BackgroundServiceTask : BackgroundService
machineName == task.LeasedByMachineName))
{
+ var updateTask = false;
var scheduleTask = serviceProvider.GetRequiredKeyedService<IScheduleTask>(task.ScheduleTaskName);
if (scheduleTask != null)
{
@@ -65,32 +70,57 @@ public class BackgroundServiceTask : BackgroundService
if (runTask)
{
- task.LastStartUtc = DateTime.UtcNow;
- try
- {
- logger.LogInformation($"Task {Name} execute");
- await scheduleTask.Execute();
- task.LastSuccessUtc = DateTime.UtcNow;
- task.LastNonSuccessEndUtc = null;
+ //claim this run atomically - when several instances race,
+ //only one wins and executes the task (no duplicated e-mails etc.)
+ var runStartUtc = DateTime.UtcNow;
+ var claimed = await scheduleTaskService.TryClaimTaskRun(task.Id, task.LastStartUtc,
+ runStartUtc, InstanceId);
+ if (claimed)
+ {
+ updateTask = true;
+ task.LastStartUtc = runStartUtc;
+ task.LeasedByInstance = InstanceId;
+ try
+ {
+ logger.LogInformation("Task {TaskName} execute", Name);
+ await scheduleTask.Execute();
+ task.LastSuccessUtc = DateTime.UtcNow;
+ task.LastNonSuccessEndUtc = null;
+ }
+ catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+ {
+ //application shutdown - do not classify as a task failure
+ throw;
+ }
+ catch (Exception exc)
+ {
+ task.LastNonSuccessEndUtc = DateTime.UtcNow;
+ task.Enabled = !task.StopOnError;
+ logger.LogError(exc,
+ "Error while running the \'{TaskScheduleTaskName}\' schedule task",
+ task.ScheduleTaskName);
+ }
}
- catch (Exception exc)
+ else
{
- task.LastNonSuccessEndUtc = DateTime.UtcNow;
- task.Enabled = !task.StopOnError;
- logger.LogError(exc,
- "Error while running the \'{TaskScheduleTaskName}\' schedule task",
- task.ScheduleTaskName);
+ //another instance executes this run - check again on the next interval
+ if (logger.IsEnabled(LogLevel.Debug))
+ logger.LogDebug("Task {TaskName} claimed by another instance, skipping", Name);
}
}
}
else
{
+ updateTask = true;
task.Enabled = !task.StopOnError;
task.LastNonSuccessEndUtc = DateTime.UtcNow;
logger.LogError("Type {TaskName} is not registered", Name);
}
- await scheduleTaskService.UpdateTask(task);
+ //persist only when this instance actually ran the task - an unconditional
+ //write would overwrite the claim/results of the winning instance with stale data
+ if (updateTask)
+ await scheduleTaskService.UpdateTask(task);
await Task.Delay(TimeSpan.FromMinutes(timeInterval), stoppingToken);
}
else
diff --git a/src/Web/Grand.Web.Common/Startup/StartupApplication.cs b/src/Web/Grand.Web.Common/Startup/StartupApplication.cs
index 943280079..73ea28c19 100644
--- a/src/Web/Grand.Web.Common/Startup/StartupApplication.cs
+++ b/src/Web/Grand.Web.Common/Startup/StartupApplication.cs
@@ -60,16 +60,24 @@ public class StartupApplication : IStartupApplication
var config = new RedisConfig();
configuration.GetSection("Redis").Bind(config);
- serviceCollection.AddSingleton<ICacheBase, MemoryCacheBase>();
-
if (config.RedisPubSubEnabled)
{
- var redis = ConnectionMultiplexer.Connect(config.RedisPubSubConnectionString);
- serviceCollection.AddSingleton(_ => redis.GetSubscriber());
- serviceCollection.AddSingleton<IMessageBus, RedisMessageBus>();
+ //AbortOnConnectFail=false lets the multiplexer keep retrying in the background
+ //instead of crashing the instance when Redis is temporarily unavailable at startup
+ var options = ConfigurationOptions.Parse(config.RedisPubSubConnectionString);
+ options.AbortOnConnectFail = false;
+ var redis = ConnectionMultiplexer.Connect(options);
+ serviceCollection.AddSingleton<IConnectionMultiplexer>(redis);
+ //single instance exposed both as the message bus and as a hosted service that
+ //owns the subscription lifecycle (start/stop) instead of the constructor
+ serviceCollection.AddSingleton<RedisMessageBus>();
+ serviceCollection.AddSingleton<IMessageBus>(sp => sp.GetRequiredService<RedisMessageBus>());
+ serviceCollection.AddHostedService(sp => sp.GetRequiredService<RedisMessageBus>());
serviceCollection.AddSingleton<ICacheBase, RedisMessageCacheManager>();
return;
}
+
+ serviceCollection.AddSingleton<ICacheBase, MemoryCacheBase>();
}
private static void RegisterContextService(IServiceCollection serviceCollection)