Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#

Fixed radarr download loop from background health checks.

nzbdav-devc212ab20ba60
diff --git a/backend/Exceptions/UsenetArticleNotFoundException.cs b/backend/Exceptions/UsenetArticleNotFoundException.cs
index 3df9827f..0233f994 100644
--- a/backend/Exceptions/UsenetArticleNotFoundException.cs
+++ b/backend/Exceptions/UsenetArticleNotFoundException.cs
@@ -3,4 +3,5 @@
public class UsenetArticleNotFoundException(string segmentId)
: NonRetryableDownloadException($"Article with message-id {segmentId} not found.")
{
+ public string SegmentId => segmentId;
}
\ No newline at end of file
diff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.cs
index 43775b8d..70a8f011 100644
--- a/backend/Queue/QueueItemProcessor.cs
+++ b/backend/Queue/QueueItemProcessor.cs
@@ -7,6 +7,7 @@ using NzbWebDAV.Clients.Usenet.Connections;
using NzbWebDAV.Config;
using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
+using NzbWebDAV.Exceptions;
using NzbWebDAV.Extensions;
using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;
using NzbWebDAV.Queue.DeobfuscationSteps._2.GetPar2FileDescriptors;
@@ -14,6 +15,7 @@ using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;
using NzbWebDAV.Queue.FileAggregators;
using NzbWebDAV.Queue.FileProcessors;
using NzbWebDAV.Queue.PostProcessors;
+using NzbWebDAV.Services;
using NzbWebDAV.Utils;
using NzbWebDAV.Websocket;
using Serilog;
@@ -28,6 +30,7 @@ public class QueueItemProcessor(
UsenetStreamingClient usenetClient,
ConfigManager configManager,
WebsocketManager websocketManager,
+ HealthCheckService healthCheckService,
IProgress<int> progress,
CancellationToken ct
)
@@ -116,7 +119,12 @@ public class QueueItemProcessor(
var archivePassword = nzb.MetaData.GetValueOrDefault("password")?.FirstOrDefault();
var nzbFiles = nzb.Files.Where(x => x.Segments.Count > 0).ToList();
- // part 1 -- get name and size of each nzb file
+ // step 0 -- perform article existence pre-check against cache
+ // https://github.com/nzbdav-dev/nzbdav/issues/101
+ var articlesToPrecheck = nzbFiles.SelectMany(x => x.Segments).Select(x => x.MessageId.Value);
+ healthCheckService.CheckCachedMissingSegmentIds(articlesToPrecheck);
+
+ // step 1 -- get name and size of each nzb file
var part1Progress = progress
.Scale(50, 100)
.ToPercentage(nzbFiles.Count);
@@ -127,7 +135,7 @@ public class QueueItemProcessor(
var fileInfos = GetFileInfosStep.GetFileInfos(
segments, par2FileDescriptors);
- // part 2 -- perform file processing
+ // step 2 -- perform file processing
var fileProcessors = GetFileProcessors(fileInfos, archivePassword).ToList();
var part2Progress = progress
.Offset(50)
@@ -142,7 +150,7 @@ public class QueueItemProcessor(
.Select(x => x!)
.ToList();
- // part3 -- Optionally check full article existence
+ // step 3 -- Optionally check full article existence
var checkedFullHealth = false;
if (configManager.IsEnsureArticleExistenceEnabled())
{
diff --git a/backend/Queue/QueueManager.cs b/backend/Queue/QueueManager.cs
index f6e98397..9df0d818 100644
--- a/backend/Queue/QueueManager.cs
+++ b/backend/Queue/QueueManager.cs
@@ -2,6 +2,7 @@
using NzbWebDAV.Config;
using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
+using NzbWebDAV.Services;
using NzbWebDAV.Utils;
using NzbWebDAV.Websocket;
using Serilog;
@@ -17,16 +18,19 @@ public class QueueManager : IDisposable
private readonly SemaphoreSlim _semaphore = new(1, 1);
private readonly ConfigManager _configManager;
private readonly WebsocketManager _websocketManager;
+ private readonly HealthCheckService _healthCheckService;
public QueueManager(
UsenetStreamingClient usenetClient,
ConfigManager configManager,
- WebsocketManager websocketManager
+ WebsocketManager websocketManager,
+ HealthCheckService healthCheckService
)
{
_usenetClient = usenetClient;
_configManager = configManager;
_websocketManager = websocketManager;
+ _healthCheckService = healthCheckService;
_cancellationTokenSource = CancellationTokenSource
.CreateLinkedTokenSource(SigtermUtil.GetCancellationToken());
_ = ProcessQueueAsync(_cancellationTokenSource.Token);
@@ -109,7 +113,8 @@ public class QueueManager : IDisposable
var progressHook = new Progress<int>();
var task = new QueueItemProcessor(
queueItem, queueNzbContents, dbClient, _usenetClient,
- _configManager, _websocketManager, progressHook, cts.Token
+ _configManager, _websocketManager, _healthCheckService,
+ progressHook, cts.Token
).ProcessAsync();
var inProgressQueueItem = new InProgressQueueItem()
{
diff --git a/backend/Services/HealthCheckService.cs b/backend/Services/HealthCheckService.cs
index 727c2d14..03f12068 100644
--- a/backend/Services/HealthCheckService.cs
+++ b/backend/Services/HealthCheckService.cs
@@ -22,6 +22,8 @@ public class HealthCheckService
private readonly WebsocketManager _websocketManager;
private readonly CancellationToken _cancellationToken = SigtermUtil.GetCancellationToken();
+ private readonly HashSet<string> _missingSegmentIds = [];
+
public HealthCheckService
(
ConfigManager configManager,
@@ -32,6 +34,14 @@ public class HealthCheckService
_configManager = configManager;
_usenetClient = usenetClient;
_websocketManager = websocketManager;
+
+ _configManager.OnConfigChanged += (_, configEventArgs) =>
+ {
+ // when usenet host changes, clear the missing segments cache
+ if (!configEventArgs.ChangedConfig.ContainsKey("usenet.host")) return;
+ lock (_missingSegmentIds) _missingSegmentIds.Clear();
+ };
+
_ = StartMonitoringService();
}
@@ -139,11 +149,15 @@ public class HealthCheckService
}));
await dbClient.Ctx.SaveChangesAsync(ct);
}
- catch (UsenetArticleNotFoundException)
+ catch (UsenetArticleNotFoundException e)
{
- // when usenet article is missing, perform repairs
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|100");
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|done");
+ if (FilenameUtil.IsImportantFileType(davItem.Name))
+ lock (_missingSegmentIds)
+ _missingSegmentIds.Add(e.SegmentId);
+
+ // when usenet article is missing, perform repairs
await Repair(davItem, dbClient, ct);
}
}
@@ -300,4 +314,14 @@ public class HealthCheckService
);
return result;
}
+
+ public void CheckCachedMissingSegmentIds(IEnumerable<string> segmentIds)
+ {
+ lock (_missingSegmentIds)
+ {
+ foreach (var segmentId in segmentIds)
+ if (_missingSegmentIds.Contains(segmentId))
+ throw new UsenetArticleNotFoundException(segmentId);
+ }
+ }
}
\ No newline at end of file