Skip to content

Open-source eCommerce platform for C# and ASP.NET Core 10 with MongoDB. Build multi-store shops and multi-vendor marketplaces with B2B/B2C, REST API, plugins, Vue 3 and Docker.

Fix scheduler loop permanently stopping on reversible states (#787)

Krzysztof Pajak9aa8c17fee03
diff --git a/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs
index eba0394a8..5bfe5830b 100644
--- a/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs
+++ b/src/Tests/Grand.Web.Common.Tests/Infrastructure/BackgroundServiceTaskTests.cs
@@ -3,8 +3,10 @@ using Grand.Domain.Tasks;
using Grand.Infrastructure;
using Grand.Web.Common.Infrastructure;
using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
+using static Grand.Web.Common.Infrastructure.BackgroundServiceTask;
namespace Grand.Web.Common.Tests.Infrastructure;
@@ -132,6 +134,39 @@ public class BackgroundServiceTaskTests
_scheduleTaskMock.Verify(t => t.Execute(), Times.Never);
}
+ [TestMethod]
+ public async Task ExecuteAsync_UnexpectedExceptionInLoop_LogsError()
+ {
+ _scheduleTaskServiceMock.Setup(s => s.GetTaskByName(TaskName))
+ .ThrowsAsync(new InvalidOperationException("transient failure"));
+
+ var loggerMock = new Mock<ILogger<BackgroundServiceTask>>();
+ var services = new ServiceCollection();
+ services.AddSingleton(_scheduleTaskServiceMock.Object);
+ services.AddKeyedSingleton(TaskName, _scheduleTaskMock.Object);
+ services.AddSingleton(Mock.Of<IContextAccessor>());
+ services.AddSingleton(Mock.Of<IStoreContextSetter>());
+ services.AddSingleton(Mock.Of<IWorkContextSetter>());
+ services.AddSingleton(loggerMock.Object);
+ var serviceProvider = services.BuildServiceProvider();
+
+ var service = new BackgroundServiceTask(TaskName, serviceProvider);
+ using var cts = new CancellationTokenSource();
+ await service.StartAsync(cts.Token);
+
+ await WaitFor(() => loggerMock.Invocations.Any(i => i.Method.Name == "Log"));
+ cts.Cancel();
+
+ //the failure must be logged, not swallowed silently
+ loggerMock.Verify(l => l.Log(
+ LogLevel.Error,
+ It.IsAny<EventId>(),
+ It.IsAny<It.IsAnyType>(),
+ It.Is<Exception>(e => e is InvalidOperationException),
+ It.IsAny<Func<It.IsAnyType, Exception, string>>()),
+ Times.AtLeastOnce);
+ }
+
private static async Task WaitFor(Func<bool> condition)
{
for (var i = 0; i < 300; i++)
@@ -143,3 +178,102 @@ public class BackgroundServiceTaskTests
Assert.Fail("Condition was not met within the timeout");
}
}
+
+/// <summary>
+/// Pure unit tests for the scheduling decision - no timers, no DI, no BackgroundService
+/// lifecycle. This is what actually proves REL-001 is fixed: every branch resolves to a
+/// delay-and-retry, none of them can signal "stop the loop".
+/// </summary>
+[TestClass]
+public class BackgroundServiceTaskDecideTests
+{
+ private static readonly DateTime UtcNow = new(2026, 8, 16, 12, 0, 0, DateTimeKind.Utc);
+ private const string MachineName = "this-machine";
+
+ private static ScheduleTask NewTask(bool enabled = true, string leasedBy = null,
+ int timeInterval = 60, DateTime? lastStartUtc = null)
+ {
+ return new ScheduleTask {
+ Id = "1",
+ Enabled = enabled,
+ LeasedByMachineName = leasedBy,
+ TimeInterval = timeInterval,
+ LastStartUtc = lastStartUtc
+ };
+ }
+
+ [TestMethod]
+ public void Decide_TaskNotSeededYet_ReturnsRetryInOneMinute()
+ {
+ var decision = Decide(null, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.Retry, decision.Action);
+ Assert.AreEqual(1, decision.DelayMinutes);
+ }
+
+ [TestMethod]
+ public void Decide_TaskDisabled_ReturnsRetryAfterOwnInterval()
+ {
+ var task = NewTask(enabled: false, timeInterval: 15);
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.Retry, decision.Action);
+ Assert.AreEqual(15, decision.DelayMinutes);
+ }
+
+ [TestMethod]
+ public void Decide_TaskLeasedByAnotherMachine_ReturnsRetryAfterOwnInterval()
+ {
+ var task = NewTask(leasedBy: "other-machine", timeInterval: 15);
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.Retry, decision.Action);
+ Assert.AreEqual(15, decision.DelayMinutes);
+ }
+
+ [TestMethod]
+ public void Decide_TaskLeasedByOwnMachine_IsEligible()
+ {
+ var task = NewTask(leasedBy: MachineName);
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.RunNow, decision.Action);
+ }
+
+ [TestMethod]
+ public void Decide_EnabledAndNeverRunBefore_ReturnsRunNow()
+ {
+ var task = NewTask(timeInterval: 30);
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.RunNow, decision.Action);
+ Assert.AreEqual(30, decision.DelayMinutes);
+ }
+
+ [TestMethod]
+ public void Decide_EnabledAndIntervalElapsed_ReturnsRunNow()
+ {
+ var task = NewTask(timeInterval: 30, lastStartUtc: UtcNow.AddMinutes(-31));
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.RunNow, decision.Action);
+ Assert.AreEqual(30, decision.DelayMinutes);
+ }
+
+ [TestMethod]
+ public void Decide_EnabledButNotDueYet_ReturnsWaitForRemainingTimeTruncatedToWholeMinutes()
+ {
+ //10.5 minutes elapsed of a 30-minute interval -> 19.5 minutes remain
+ var task = NewTask(timeInterval: 30, lastStartUtc: UtcNow.AddSeconds(-(10 * 60 + 30)));
+
+ var decision = Decide(task, MachineName, UtcNow);
+
+ Assert.AreEqual(ScheduleAction.WaitForNextRun, decision.Action);
+ Assert.AreEqual(19, decision.DelayMinutes); //truncated, same as the original code
+ }
+}
diff --git a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
index cc03ed0d6..09985c4f5 100644
--- a/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
+++ b/src/Web/Grand.Web.Common/Infrastructure/BackgroundServiceTask.cs
@@ -1,10 +1,13 @@
-using Grand.Business.Core.Interfaces.System.ScheduleTasks;
+using System.Runtime.CompilerServices;
+using Grand.Business.Core.Interfaces.System.ScheduleTasks;
using Grand.Domain.Tasks;
using Grand.Infrastructure;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
+[assembly: InternalsVisibleTo("Grand.Web.Common.Tests")]
+
namespace Grand.Web.Common.Infrastructure;
public class BackgroundServiceTask : BackgroundService
@@ -22,116 +25,145 @@ public class BackgroundServiceTask : BackgroundService
_serviceProvider = serviceProvider;
}
+ internal enum ScheduleAction
+ {
+ //task missing, disabled, or leased to another instance - all reversible, poll again later
+ Retry,
+ //own lease (or none), enabled, and due - claim and execute
+ RunNow,
+ //own lease, enabled, but not due yet - sleep until it is
+ WaitForNextRun
+ }
+
+ internal readonly record struct ScheduleDecision(ScheduleAction Action, int DelayMinutes);
+
+ /// <summary>
+ /// Pure decision of what the loop should do this iteration and how long to sleep
+ /// afterward. Deliberately has no side effects and no terminal outcome - every branch
+ /// resolves to a delay-and-retry, never to "stop the loop" - so the reversible states
+ /// (task not seeded yet, disabled, leased elsewhere) can never end the loop for good.
+ /// </summary>
+ internal static ScheduleDecision Decide(ScheduleTask task, string machineName, DateTime utcNow)
+ {
+ if (task == null)
+ return new ScheduleDecision(ScheduleAction.Retry, 1);
+
+ var timeInterval = task.TimeInterval > 0 ? task.TimeInterval : 1;
+
+ var eligible = task.Enabled &&
+ (string.IsNullOrEmpty(task.LeasedByMachineName) || machineName == task.LeasedByMachineName);
+ if (!eligible)
+ return new ScheduleDecision(ScheduleAction.Retry, timeInterval);
+
+ if (task.LastStartUtc.HasValue)
+ {
+ var nextRunUtc = task.LastStartUtc.Value.AddMinutes(task.TimeInterval);
+ if (utcNow < nextRunUtc)
+ return new ScheduleDecision(ScheduleAction.WaitForNextRun, (int)(nextRunUtc - utcNow).TotalMinutes);
+ }
+
+ return new ScheduleDecision(ScheduleAction.RunNow, timeInterval);
+ }
+
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
+ //resolved once from the root provider (not the per-iteration scope) so it also
+ //survives to log failures raised before/around a scope, e.g. in the outer catch
+ var logger = _serviceProvider.GetService<ILogger<BackgroundServiceTask>>();
+
while (!stoppingToken.IsCancellationRequested)
try
{
using var scope = _serviceProvider.CreateScope();
var serviceProvider = scope.ServiceProvider;
- var logger = serviceProvider.GetService<ILogger<BackgroundServiceTask>>();
var scheduleTaskService = serviceProvider.GetService<IScheduleTaskService>();
var task = await scheduleTaskService.GetTaskByName(Name);
+ var decision = Decide(task, Environment.MachineName, DateTime.UtcNow);
+
if (task == null)
- {
logger.LogInformation("Task {TaskName} is not exists in the database", Name);
- break;
- }
+ else if (decision.Action != ScheduleAction.Retry)
+ await RunTask(serviceProvider, scheduleTaskService, task,
+ decision.Action == ScheduleAction.RunNow, logger, stoppingToken);
- var machineName = Environment.MachineName;
- var timeInterval = task.TimeInterval > 0 ? task.TimeInterval : 1;
- if (task.Enabled && (string.IsNullOrEmpty(task.LeasedByMachineName) ||
- machineName == task.LeasedByMachineName))
- {
+ //every branch above falls through to here - the loop always sleeps and
+ //retries, it never exits on its own (only cancellation ends it)
+ await Task.Delay(TimeSpan.FromMinutes(decision.DelayMinutes), stoppingToken);
+ }
+ catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+ {
+ //application shutdown - not a task failure
+ }
+ catch (Exception exc)
+ {
+ logger.LogError(exc, "Unhandled error in the background loop for task {TaskName}", Name);
+ await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
+ }
+ }
+
+ private async Task RunTask(IServiceProvider serviceProvider, IScheduleTaskService scheduleTaskService,
+ ScheduleTask task, bool due, ILogger logger, CancellationToken stoppingToken)
+ {
+ var updateTask = false;
+ var scheduleTask = serviceProvider.GetRequiredKeyedService<IScheduleTask>(task.ScheduleTaskName);
+ if (scheduleTask != null)
+ {
+ //assign current customer (background task) / current store (from task)
+ await WorkContext(serviceProvider, task);
+
+ if (!due)
+ return;
- var updateTask = false;
- var scheduleTask = serviceProvider.GetRequiredKeyedService<IScheduleTask>(task.ScheduleTaskName);
- if (scheduleTask != null)
- {
- //assign current customer (background task) / current store (from task)
- await WorkContext(serviceProvider, task);
- var runTask = true;
- if (task.LastStartUtc.HasValue)
- {
- var dateTimeNow = DateTime.UtcNow;
- if (dateTimeNow < task.LastStartUtc.Value.AddMinutes(task.TimeInterval))
- {
- runTask = false;
- timeInterval =
- (int)(task.LastStartUtc.Value.AddMinutes(task.TimeInterval) - dateTimeNow)
- .TotalMinutes;
- }
- else
- {
- runTask = true;
- timeInterval = task.TimeInterval > 0 ? task.TimeInterval : 1;
- }
- }
-
- if (runTask)
- {
- //claim this run atomically - when several instances race,
- //only one wins and executes the task (no duplicated e-mails etc.)
- var runStartUtc = DateTime.UtcNow;
- var claimed = await scheduleTaskService.TryClaimTaskRun(task.Id, task.LastStartUtc,
- runStartUtc, InstanceId);
- if (claimed)
- {
- updateTask = true;
- task.LastStartUtc = runStartUtc;
- task.LeasedByInstance = InstanceId;
- try
- {
- logger.LogInformation("Task {TaskName} execute", Name);
- await scheduleTask.Execute();
- task.LastSuccessUtc = DateTime.UtcNow;
- task.LastNonSuccessEndUtc = null;
- }
- catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
- {
- //application shutdown - do not classify as a task failure
- throw;
- }
- catch (Exception exc)
- {
- task.LastNonSuccessEndUtc = DateTime.UtcNow;
- task.Enabled = !task.StopOnError;
- logger.LogError(exc,
- "Error while running the \'{TaskScheduleTaskName}\' schedule task",
- task.ScheduleTaskName);
- }
- }
- else
- {
- //another instance executes this run - check again on the next interval
- if (logger.IsEnabled(LogLevel.Debug))
- logger.LogDebug("Task {TaskName} claimed by another instance, skipping", Name);
- }
- }
- }
- else
- {
- updateTask = true;
- task.Enabled = !task.StopOnError;
- task.LastNonSuccessEndUtc = DateTime.UtcNow;
- logger.LogError("Type {TaskName} is not registered", Name);
- }
-
- //persist only when this instance actually ran the task - an unconditional
- //write would overwrite the claim/results of the winning instance with stale data
- if (updateTask)
- await scheduleTaskService.UpdateTask(task);
- await Task.Delay(TimeSpan.FromMinutes(timeInterval), stoppingToken);
+ //claim this run atomically - when several instances race,
+ //only one wins and executes the task (no duplicated e-mails etc.)
+ var runStartUtc = DateTime.UtcNow;
+ var claimed = await scheduleTaskService.TryClaimTaskRun(task.Id, task.LastStartUtc,
+ runStartUtc, InstanceId);
+ if (claimed)
+ {
+ updateTask = true;
+ task.LastStartUtc = runStartUtc;
+ task.LeasedByInstance = InstanceId;
+ try
+ {
+ logger.LogInformation("Task {TaskName} execute", Name);
+ await scheduleTask.Execute();
+ task.LastSuccessUtc = DateTime.UtcNow;
+ task.LastNonSuccessEndUtc = null;
}
- else
+ catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
- break;
+ //application shutdown - do not classify as a task failure
+ throw;
+ }
+ catch (Exception exc)
+ {
+ task.LastNonSuccessEndUtc = DateTime.UtcNow;
+ task.Enabled = !task.StopOnError;
+ logger.LogError(exc,
+ "Error while running the \'{TaskScheduleTaskName}\' schedule task",
+ task.ScheduleTaskName);
}
}
- catch (Exception)
+ else
{
- await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
+ //another instance executes this run - check again on the next interval
+ if (logger.IsEnabled(LogLevel.Debug))
+ logger.LogDebug("Task {TaskName} claimed by another instance, skipping", Name);
}
+ }
+ else
+ {
+ updateTask = true;
+ task.Enabled = !task.StopOnError;
+ task.LastNonSuccessEndUtc = DateTime.UtcNow;
+ logger.LogError("Type {TaskName} is not registered", Name);
+ }
+
+ //persist only when this instance actually ran the task - an unconditional
+ //write would overwrite the claim/results of the winning instance with stale data
+ if (updateTask)
+ await scheduleTaskService.UpdateTask(task);
}
private static async Task WorkContext(IServiceProvider serviceProvider, ScheduleTask scheduleTask)