Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using NzbWebDAV.Clients.Usenet;
using NzbWebDAV.Config;
using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
using NzbWebDAV.Exceptions;
using NzbWebDAV.Extensions;
using NzbWebDAV.Queue.PostProcessors;
using NzbWebDAV.Utils;
using NzbWebDAV.Websocket;
using Serilog;
namespace NzbWebDAV.Services;
/// <summary>
/// This service monitors for health checks
/// </summary>
public class HealthCheckService : BackgroundService
{
private readonly ConfigManager _configManager;
private readonly INntpClient _usenetClient;
private readonly WebsocketManager _websocketManager;
private static readonly HashSet<string> _missingSegmentIds = [];
public HealthCheckService
(
ConfigManager configManager,
UsenetStreamingClient usenetClient,
WebsocketManager websocketManager
)
{
_configManager = configManager;
_usenetClient = usenetClient;
_websocketManager = websocketManager;
_configManager.OnConfigChanged += (_, configEventArgs) =>
{
// when usenet host changes, clear the missing segments cache
if (!configEventArgs.ChangedConfig.ContainsKey("usenet.host")) return;
lock (_missingSegmentIds) _missingSegmentIds.Clear();
};
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
// if the repair-job is disabled, then don't do anything
if (!_configManager.IsRepairJobEnabled())
{
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
continue;
}
// get concurrency
var concurrency = _configManager.GetUsenetProviderConfig().TotalPooledConnections;
using var cts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
// get the davItem to health-check
await using var dbContext = new DavDatabaseContext();
var dbClient = new DavDatabaseClient(dbContext);
var currentDateTime = DateTimeOffset.UtcNow;
var davItem = await GetHealthCheckQueueItems(dbClient)
.Where(x => x.NextHealthCheck == null || x.NextHealthCheck < currentDateTime)
.FirstOrDefaultAsync(cts.Token).ConfigureAwait(false);
// if there is no item to health-check, don't do anything
if (davItem == null)
{
await Task.Delay(TimeSpan.FromSeconds(5), cts.Token).ConfigureAwait(false);
continue;
}
// perform the health check
await PerformHealthCheck(davItem, dbClient, concurrency, cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (SigtermUtil.IsSigtermTriggered())
{
// OperationCanceledException is expected on sigterm
return;
}
catch (Exception e)
{
Log.Error(e, $"Unexpected error performing background health checks: {e.Message}");
await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken).ConfigureAwait(false);
}
}
}
public static IOrderedQueryable<DavItem> GetHealthCheckQueueItems(DavDatabaseClient dbClient)
{
return GetHealthCheckQueueItemsQuery(dbClient)
.OrderBy(x => x.NextHealthCheck)
.ThenByDescending(x => x.ReleaseDate)
.ThenBy(x => x.Id);
}
public static IQueryable<DavItem> GetHealthCheckQueueItemsQuery(DavDatabaseClient dbClient)
{
return dbClient.Ctx.Items
.Where(x => x.Type == DavItem.ItemType.UsenetFile)
.Where(x => x.HistoryItemId == null);
}
private async Task PerformHealthCheck
(
DavItem davItem,
DavDatabaseClient dbClient,
int concurrency,
CancellationToken ct
)
{
try
{
// update the release date, if null
var segments = await GetAllSegments(davItem, dbClient, ct).ConfigureAwait(false);
if (davItem.ReleaseDate == null) await UpdateReleaseDate(davItem, segments, ct).ConfigureAwait(false);
// setup progress tracking
var progressHook = new Progress<int>();
var debounce = DebounceUtil.CreateDebounce(TimeSpan.FromMilliseconds(200));
progressHook.ProgressChanged += (_, progress) =>
{
var message = $"{davItem.Id}|{progress}";
debounce(() => _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, message));
};
// perform health check
var progress = progressHook.ToPercentage(segments.Count);
await _usenetClient.CheckAllSegmentsAsync(segments, concurrency, progress, ct).ConfigureAwait(false);
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|100");
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|done");
// update the database
davItem.LastHealthCheck = DateTimeOffset.UtcNow;
davItem.NextHealthCheck = davItem.ReleaseDate + 2 * (davItem.LastHealthCheck - davItem.ReleaseDate);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = DateTimeOffset.UtcNow,
Result = HealthCheckResult.HealthResult.Healthy,
RepairStatus = HealthCheckResult.RepairAction.None,
Message = "File is healthy."
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
}
catch (UsenetArticleNotFoundException e)
{
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|100");
_ = _websocketManager.SendMessage(WebsocketTopic.HealthItemProgress, $"{davItem.Id}|done");
if (FilenameUtil.IsImportantFileType(davItem.Name))
lock (_missingSegmentIds)
_missingSegmentIds.Add(e.SegmentId);
// when usenet article is missing, perform repairs
await Repair(davItem, dbClient, ct).ConfigureAwait(false);
}
}
private async Task UpdateReleaseDate(DavItem davItem, List<string> segments, CancellationToken ct)
{
var firstSegmentId = StringUtil.EmptyToNull(segments.FirstOrDefault());
if (firstSegmentId == null) return;
var articleHeadersResponse = await _usenetClient.HeadAsync(firstSegmentId, ct).ConfigureAwait(false);
var articleHeaders = articleHeadersResponse.ArticleHeaders!;
davItem.ReleaseDate = articleHeaders.Date;
}
private async Task<List<string>> GetAllSegments(DavItem davItem, DavDatabaseClient dbClient, CancellationToken ct)
{
if (davItem.SubType == DavItem.ItemSubType.NzbFile)
{
var nzbFile = await dbClient.GetDavNzbFileAsync(davItem, ct).ConfigureAwait(false);
return nzbFile?.SegmentIds?.ToList() ?? [];
}
if (davItem.SubType == DavItem.ItemSubType.RarFile)
{
var rarFile = await dbClient.GetDavRarFileAsync(davItem, ct).ConfigureAwait(false);
return rarFile?.RarParts?.SelectMany(x => x.SegmentIds)?.ToList() ?? [];
}
if (davItem.SubType == DavItem.ItemSubType.MultipartFile)
{
var multipartFile = await dbClient.GetDavMultipartFileAsync(davItem, ct).ConfigureAwait(false);
return multipartFile?.Metadata?.FileParts?.SelectMany(x => x.SegmentIds)?.ToList() ?? [];
}
return [];
}
private async Task Repair(DavItem davItem, DavDatabaseClient dbClient, CancellationToken ct)
{
try
{
// if the file pattern has been marked as ignored,
// then don't bother trying to repair it. We can simply delete it.
var blocklistedFiles = _configManager.GetBlocklistedFiles();
if (BlocklistedFilePostProcessor.MatchesAnyPattern(davItem.Name, blocklistedFiles))
{
dbClient.Ctx.Items.Remove(davItem);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = DateTimeOffset.UtcNow,
Result = HealthCheckResult.HealthResult.Unhealthy,
RepairStatus = HealthCheckResult.RepairAction.Deleted,
Message = string.Join(" ", [
"File had missing articles.",
"Filename pattern is marked in settings as an ignored (unwanted) file.",
"Deleted file."
])
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
return;
}
// if the unhealthy item is unlinked/orphaned,
// then we can simply delete it.
var symlinkOrStrmPath = OrganizedLinksUtil.GetLink(davItem, _configManager);
if (symlinkOrStrmPath == null)
{
dbClient.Ctx.Items.Remove(davItem);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = DateTimeOffset.UtcNow,
Result = HealthCheckResult.HealthResult.Unhealthy,
RepairStatus = HealthCheckResult.RepairAction.Deleted,
Message = string.Join(" ", [
"File had missing articles.",
"Could not find corresponding symlink or strm-file within Library Dir.",
"Deleted file."
])
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
return;
}
// if the unhealthy item is linked within the organized media-library
// then we must find the corresponding arr instance and trigger a new search.
var linkType = symlinkOrStrmPath.ToLower().EndsWith("strm") ? "strm-file" : "symlink";
foreach (var arrClient in _configManager.GetArrConfig().GetArrClients())
{
var rootFolders = await arrClient.GetRootFolders().ConfigureAwait(false);
if (!rootFolders.Any(x => symlinkOrStrmPath.StartsWith(x.Path!))) continue;
// if we found a corresponding arr instance,
// then remove and search.
if (await arrClient.RemoveAndSearch(symlinkOrStrmPath).ConfigureAwait(false))
{
dbClient.Ctx.Items.Remove(davItem);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = DateTimeOffset.UtcNow,
Result = HealthCheckResult.HealthResult.Unhealthy,
RepairStatus = HealthCheckResult.RepairAction.Repaired,
Message = string.Join(" ", [
"File had missing articles.",
$"Corresponding {linkType} found within Library Dir.",
"Triggered new Arr search."
])
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
return;
}
// if we could not find corresponding media-item to remove-and-search
// within the found arr instance, then break out of this loop so that
// we can fall back to the behavior below of deleting both the link-file
// and the dav-item.
break;
}
// if we could not find a corresponding arr instance
// then we can delete both the item and the link-file.
await Task.Run(() => File.Delete(symlinkOrStrmPath)).ConfigureAwait(false);
dbClient.Ctx.Items.Remove(davItem);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = DateTimeOffset.UtcNow,
Result = HealthCheckResult.HealthResult.Unhealthy,
RepairStatus = HealthCheckResult.RepairAction.Deleted,
Message = string.Join(" ", [
"File had missing articles.",
$"Corresponding {linkType} found within Library Dir.",
"Could not find corresponding Radarr/Sonarr media-item to trigger a new search.",
$"Deleted the webdav-file and {linkType}."
])
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
}
catch (Exception e)
{
// if an error is encountered during repairs,
// then mark the item as unhealthy, and check again in a day.
var utcNow = DateTimeOffset.UtcNow;
davItem.LastHealthCheck = utcNow;
davItem.NextHealthCheck = utcNow + TimeSpan.FromDays(1);
dbClient.Ctx.HealthCheckResults.Add(SendStatus(new HealthCheckResult()
{
Id = Guid.NewGuid(),
DavItemId = davItem.Id,
Path = davItem.Path,
CreatedAt = utcNow,
Result = HealthCheckResult.HealthResult.Unhealthy,
RepairStatus = HealthCheckResult.RepairAction.ActionNeeded,
Message = $"Error performing file repair: {e.Message}"
}));
await dbClient.Ctx.SaveChangesAsync(ct).ConfigureAwait(false);
}
}
private HealthCheckResult SendStatus(HealthCheckResult result)
{
_ = _websocketManager.SendMessage
(
WebsocketTopic.HealthItemStatus,
$"{result.DavItemId}|{(int)result.Result}|{(int)result.RepairStatus}"
);
return result;
}
public static void CheckCachedMissingSegmentIds(IEnumerable<string> segmentIds)
{
lock (_missingSegmentIds)
{
foreach (var segmentId in segmentIds)
if (_missingSegmentIds.Contains(segmentId))
throw new UsenetArticleNotFoundException(segmentId);
}
}
}