Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Added strm-to-symlink backend task.
nzbdav-dev62001732b60f
diff --git a/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.cs b/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.csnew file mode 100644index 00000000..ba844c83--- /dev/null+++ b/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.cs@@ -0,0 +1,23 @@+using Microsoft.AspNetCore.Mvc;+using NzbWebDAV.Config;+using NzbWebDAV.Database;+using NzbWebDAV.Tasks;+using NzbWebDAV.Websocket;++namespace NzbWebDAV.Api.Controllers.ConvertStrmToSymlinks;++[ApiController]+[Route("api/convert-strm-to-symlinks")]+public class ConvertStrmToSymlinks(+ ConfigManager configManager,+ DavDatabaseClient dbClient,+ WebsocketManager websocketManager+) : BaseApiController+{+ protected override async Task<IActionResult> HandleRequest()+ {+ var task = new StrmToSymlinksTask(configManager, dbClient, websocketManager);+ var executed = await task.Execute();+ return Ok(executed);+ }+}\ No newline at end of filediff --git a/backend/Database/Models/DavItem.cs b/backend/Database/Models/DavItem.csindex faf083bc..b0623dbc 100644--- a/backend/Database/Models/DavItem.cs+++ b/backend/Database/Models/DavItem.cs@@ -1,4 +1,5 @@using System.Text.Json.Serialization;+using NzbWebDAV.Extensions;namespace NzbWebDAV.Database.Models;@@ -32,7 +33,7 @@ public class DavItemreturn new DavItem(){Id = id,- IdPrefix = id.ToString()[..5],+ IdPrefix = id.GetFiveLengthPrefix(),CreatedAt = DateTime.Now,ParentId = parent.Id,Name = name,diff --git a/backend/Extensions/GuidExtensions.cs b/backend/Extensions/GuidExtensions.csnew file mode 100644index 00000000..bbfe089a--- /dev/null+++ b/backend/Extensions/GuidExtensions.cs@@ -0,0 +1,11 @@+// ReSharper disable InconsistentNaming++namespace NzbWebDAV.Extensions;++public static class GuidExtensions+{+ public static string GetFiveLengthPrefix(this Guid guid)+ {+ return guid.ToString()[..5];+ }+}\ No newline at end of filediff --git a/backend/Extensions/IEnumerableExtensions.cs b/backend/Extensions/IEnumerableExtensions.csnew file mode 100644index 00000000..f650a7d9--- /dev/null+++ b/backend/Extensions/IEnumerableExtensions.cs@@ -0,0 +1,31 @@+// ReSharper disable InconsistentNaming++namespace NzbWebDAV.Extensions;++public static class IEnumerableExtensions+{+ public static IEnumerable<List<T>> ToBatches<T>(this IEnumerable<T> items, int batchSize)+ {+ if (items == null)+ throw new ArgumentNullException(nameof(items));+ if (batchSize <= 0)+ throw new ArgumentOutOfRangeException(+ nameof(batchSize),+ "Batch size must be greater than zero."+ );++ var batch = new List<T>(batchSize);+ foreach (var item in items)+ {+ batch.Add(item);+ if (batch.Count >= batchSize)+ {+ yield return batch;+ batch = new List<T>(batchSize);+ }+ }++ if (batch.Count > 0)+ yield return batch;+ }+}\ No newline at end of filediff --git a/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs b/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.csindex 37734ad2..c90d63b1 100644--- a/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs+++ b/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs@@ -5,7 +5,6 @@ using NzbWebDAV.Database;using NzbWebDAV.Database.Models;using NzbWebDAV.Utils;using NzbWebDAV.WebDav;-using Serilog;namespace NzbWebDAV.Queue.PostProcessors;@@ -46,11 +45,11 @@ public class CreateStrmFilesPostProcessor(ConfigManager configManager, DavDataba{var baseUrl = configManager.GetBaseUrl();if (baseUrl.EndsWith('/')) baseUrl = baseUrl.TrimEnd('/');- var pathUrl = DatabaseStoreSymlinkFile.GetTargetPath(davItem, "", '/');+ var pathUrl = DatabaseStoreSymlinkFile.GetTargetPath(davItem.Id, "", '/');if (pathUrl.StartsWith('/')) pathUrl = pathUrl.TrimStart('/');var strmKey = configManager.GetStrmKey();var downloadKey = GetWebdavItemRequest.GenerateDownloadKey(strmKey, pathUrl);- var extension = Path.GetExtension(davItem.Name).ToLowerInvariant().TrimStart('.');+ var extension = Path.GetExtension(davItem.Name).ToLower().TrimStart('.');return $"{baseUrl}/view/{pathUrl}?downloadKey={downloadKey}&extension={extension}";}}\ No newline at end of filediff --git a/backend/Tasks/StrmToSymlinksTask.cs b/backend/Tasks/StrmToSymlinksTask.csnew file mode 100644index 00000000..ac902f44--- /dev/null+++ b/backend/Tasks/StrmToSymlinksTask.cs@@ -0,0 +1,106 @@+using System.Web;+using Microsoft.EntityFrameworkCore;+using NzbWebDAV.Config;+using NzbWebDAV.Database;+using NzbWebDAV.Extensions;+using NzbWebDAV.Utils;+using NzbWebDAV.WebDav;+using NzbWebDAV.Websocket;+using Serilog;++namespace NzbWebDAV.Tasks;++public class StrmToSymlinksTask(+ ConfigManager configManager,+ DavDatabaseClient dbClient,+ WebsocketManager websocketManager+) : BaseTask+{+ protected override async Task ExecuteInternal()+ {+ try+ {+ var ct = SigtermUtil.GetCancellationToken();+ await ConvertAllStrmFilesToSymlinks(ct);+ }+ catch (Exception e)+ {+ Report($"Failed: {e.Message}");+ Log.Error(e, "Failed to convert *.strm files to symlinks.");+ }+ }++ private async Task ConvertAllStrmFilesToSymlinks(CancellationToken token)+ {+ var completedCount = 0;+ var debounce = DebounceUtil.CreateDebounce(TimeSpan.FromMilliseconds(200));+ var batches = OrganizedLinksUtil.GetLibraryDavItemLinks(configManager)+ .Where(x => x.SymlinkOrStrmInfo is SymlinkAndStrmUtil.StrmInfo)+ .ToBatches(batchSize: 100);++ ReportProgress("Scanning library for strm files...", completedCount);+ foreach (var batch in batches)+ await ConvertBatchOfStrmFilesToSymlinks(batch, OnItemCompleted, token);+ ReportProgress("Done!", completedCount);+ return;++ void OnItemCompleted()+ {+ completedCount++;+ debounce(() => ReportProgress("Scanning library for strm files...", completedCount));+ }+ }++ private async Task ConvertBatchOfStrmFilesToSymlinks+ (+ List<OrganizedLinksUtil.DavItemLink> batch,+ Action onItemCompleted,+ CancellationToken token+ )+ {+ var items = batch+ .Select(x => new { Link = x, Extension = GetExtension(x) })+ .ToList();+ var davItemsToFetch = items+ .Where(x => x.Extension is null)+ .Select(x => x.Link.DavItemId)+ .ToList();+ var davItems = await dbClient.Ctx.Items+ .Where(x => davItemsToFetch.Contains(x.Id))+ .ToDictionaryAsync(x => x.Id, x => x, token);+ var itemsWithExtension = items+ .Select(x => new+ {+ Link = x.Link,+ Extension = x.Extension ?? Path.GetExtension(davItems[x.Link.DavItemId].Name).TrimStart('.'),+ })+ .ToList();++ var mountDir = configManager.GetRcloneMountDir();+ foreach (var item in itemsWithExtension)+ {+ var symlinkPath = PathUtil.ReplaceExtension(item.Link.LinkPath, item.Extension);+ var symlinkTarget = DatabaseStoreSymlinkFile.GetTargetPath(item.Link.DavItemId, mountDir);+ File.CreateSymbolicLink(symlinkPath, symlinkTarget);+ File.Delete(item.Link.LinkPath);+ onItemCompleted?.Invoke();+ }+ }++ private string? GetExtension(OrganizedLinksUtil.DavItemLink link)+ {+ if (link.SymlinkOrStrmInfo is not SymlinkAndStrmUtil.StrmInfo strmInfo) return null;+ var queryParams = HttpUtility.ParseQueryString(new Uri(strmInfo.TargetUrl).Query);+ return queryParams.Get("extension");+ }++ private void Report(string message)+ {+ _ = websocketManager.SendMessage(WebsocketTopic.StrmToSymlinksTaskProgress, message);+ }++ private void ReportProgress(string message, int completedCount)+ {+ Report($"{message}\nConverted: {completedCount} strm file(s) to symlinks.");+ }+}\ No newline at end of filediff --git a/backend/Utils/OrganizedLinksUtil.cs b/backend/Utils/OrganizedLinksUtil.csindex 49b1770f..afdecb9a 100644--- a/backend/Utils/OrganizedLinksUtil.cs+++ b/backend/Utils/OrganizedLinksUtil.cs@@ -108,7 +108,8 @@ public static class OrganizedLinksUtilreturn new DavItemLink(){LinkPath = symlinkInfo.SymlinkPath,- DavItemId = Guid.Parse(guid)+ DavItemId = Guid.Parse(guid),+ SymlinkOrStrmInfo = symlinkInfo};}@@ -121,7 +122,8 @@ public static class OrganizedLinksUtilreturn new DavItemLink(){LinkPath = strmInfo.StrmPath,- DavItemId = Guid.Parse(guid)+ DavItemId = Guid.Parse(guid),+ SymlinkOrStrmInfo = strmInfo};}@@ -129,5 +131,6 @@ public static class OrganizedLinksUtil{public string LinkPath; // Path to either a symlink or strm file.public Guid DavItemId;+ public SymlinkAndStrmUtil.ISymlinkOrStrmInfo SymlinkOrStrmInfo;}}\ No newline at end of filediff --git a/backend/Utils/PathUtil.cs b/backend/Utils/PathUtil.csindex 0e922bb3..8f382906 100644--- a/backend/Utils/PathUtil.cs+++ b/backend/Utils/PathUtil.cs@@ -9,4 +9,12 @@ public class PathUtil? GetAllParentDirectories(directoryName).Prepend(directoryName): [];}++ public static string ReplaceExtension(string path, string newExtensions)+ {+ var directoryName = Path.GetDirectoryName(path);+ var filenameWithoutExtension = Path.GetFileNameWithoutExtension(path);+ var newFilename = $"{filenameWithoutExtension}.{newExtensions.TrimStart('.')}";+ return Path.Join(directoryName, newFilename);+ }}\ No newline at end of filediff --git a/backend/WebDav/DatabaseStoreSymlinkFile.cs b/backend/WebDav/DatabaseStoreSymlinkFile.csindex 6031cabb..00f7602b 100644--- a/backend/WebDav/DatabaseStoreSymlinkFile.cs+++ b/backend/WebDav/DatabaseStoreSymlinkFile.cs@@ -1,6 +1,7 @@using System.Text;using NzbWebDAV.Config;using NzbWebDAV.Database.Models;+using NzbWebDAV.Extensions;using NzbWebDAV.WebDav.Base;namespace NzbWebDAV.WebDav;@@ -21,16 +22,16 @@ public class DatabaseStoreSymlinkFile(DavItem davFile, ConfigManager configManagprivate string GetTargetPath(){- return GetTargetPath(davFile, configManager.GetRcloneMountDir());+ return GetTargetPath(davFile.Id, configManager.GetRcloneMountDir());}- public static string GetTargetPath(DavItem davFile, string mountDir, char? pathSeparator = null)+ public static string GetTargetPath(Guid davItemId, string mountDir, char? pathSeparator = null){- var pathParts = davFile.IdPrefix+ var pathParts = davItemId.GetFiveLengthPrefix().Select(x => x.ToString()).Prepend(DavItem.IdsFolder.Name).Prepend(mountDir)- .Append(davFile.Id.ToString())+ .Append(davItemId.ToString()).ToArray();return string.Join(pathSeparator ?? Path.DirectorySeparatorChar, pathParts);}diff --git a/backend/Websocket/WebsocketTopic.cs b/backend/Websocket/WebsocketTopic.csindex 7554e376..a469b00f 100644--- a/backend/Websocket/WebsocketTopic.cs+++ b/backend/Websocket/WebsocketTopic.cs@@ -6,6 +6,7 @@ public class WebsocketTopicpublic static readonly WebsocketTopic UsenetConnections = new("cxs", TopicType.State);public static readonly WebsocketTopic SymlinkTaskProgress = new("stp", TopicType.State);public static readonly WebsocketTopic CleanupTaskProgress = new("ctp", TopicType.State);+ public static readonly WebsocketTopic StrmToSymlinksTaskProgress = new("st2sy", TopicType.State);public static readonly WebsocketTopic QueueItemStatus = new("qs", TopicType.State);public static readonly WebsocketTopic QueueItemProgress = new("qp", TopicType.State);public static readonly WebsocketTopic HealthItemStatus = new("hs", TopicType.State);