Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#

Added strm-to-symlink backend task.

nzbdav-dev62001732b60f
diff --git a/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.cs b/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.cs
new file mode 100644
index 00000000..ba844c83
--- /dev/null
+++ b/backend/Api/Controllers/ConvertStrmToSymlinks/ConvertStrmToSymlinksController.cs
@@ -0,0 +1,23 @@
+using Microsoft.AspNetCore.Mvc;
+using NzbWebDAV.Config;
+using NzbWebDAV.Database;
+using NzbWebDAV.Tasks;
+using NzbWebDAV.Websocket;
+
+namespace NzbWebDAV.Api.Controllers.ConvertStrmToSymlinks;
+
+[ApiController]
+[Route("api/convert-strm-to-symlinks")]
+public class ConvertStrmToSymlinks(
+ ConfigManager configManager,
+ DavDatabaseClient dbClient,
+ WebsocketManager websocketManager
+) : BaseApiController
+{
+ protected override async Task<IActionResult> HandleRequest()
+ {
+ var task = new StrmToSymlinksTask(configManager, dbClient, websocketManager);
+ var executed = await task.Execute();
+ return Ok(executed);
+ }
+}
\ No newline at end of file
diff --git a/backend/Database/Models/DavItem.cs b/backend/Database/Models/DavItem.cs
index faf083bc..b0623dbc 100644
--- a/backend/Database/Models/DavItem.cs
+++ b/backend/Database/Models/DavItem.cs
@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
+using NzbWebDAV.Extensions;
namespace NzbWebDAV.Database.Models;
@@ -32,7 +33,7 @@ public class DavItem
return new DavItem()
{
Id = id,
- IdPrefix = id.ToString()[..5],
+ IdPrefix = id.GetFiveLengthPrefix(),
CreatedAt = DateTime.Now,
ParentId = parent.Id,
Name = name,
diff --git a/backend/Extensions/GuidExtensions.cs b/backend/Extensions/GuidExtensions.cs
new file mode 100644
index 00000000..bbfe089a
--- /dev/null
+++ b/backend/Extensions/GuidExtensions.cs
@@ -0,0 +1,11 @@
+// ReSharper disable InconsistentNaming
+
+namespace NzbWebDAV.Extensions;
+
+public static class GuidExtensions
+{
+ public static string GetFiveLengthPrefix(this Guid guid)
+ {
+ return guid.ToString()[..5];
+ }
+}
\ No newline at end of file
diff --git a/backend/Extensions/IEnumerableExtensions.cs b/backend/Extensions/IEnumerableExtensions.cs
new file mode 100644
index 00000000..f650a7d9
--- /dev/null
+++ b/backend/Extensions/IEnumerableExtensions.cs
@@ -0,0 +1,31 @@
+// ReSharper disable InconsistentNaming
+
+namespace NzbWebDAV.Extensions;
+
+public static class IEnumerableExtensions
+{
+ public static IEnumerable<List<T>> ToBatches<T>(this IEnumerable<T> items, int batchSize)
+ {
+ if (items == null)
+ throw new ArgumentNullException(nameof(items));
+ if (batchSize <= 0)
+ throw new ArgumentOutOfRangeException(
+ nameof(batchSize),
+ "Batch size must be greater than zero."
+ );
+
+ var batch = new List<T>(batchSize);
+ foreach (var item in items)
+ {
+ batch.Add(item);
+ if (batch.Count >= batchSize)
+ {
+ yield return batch;
+ batch = new List<T>(batchSize);
+ }
+ }
+
+ if (batch.Count > 0)
+ yield return batch;
+ }
+}
\ No newline at end of file
diff --git a/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs b/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs
index 37734ad2..c90d63b1 100644
--- a/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs
+++ b/backend/Queue/PostProcessors/CreateStrmFilesPostProcessor.cs
@@ -5,7 +5,6 @@ using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
using NzbWebDAV.Utils;
using NzbWebDAV.WebDav;
-using Serilog;
namespace NzbWebDAV.Queue.PostProcessors;
@@ -46,11 +45,11 @@ public class CreateStrmFilesPostProcessor(ConfigManager configManager, DavDataba
{
var baseUrl = configManager.GetBaseUrl();
if (baseUrl.EndsWith('/')) baseUrl = baseUrl.TrimEnd('/');
- var pathUrl = DatabaseStoreSymlinkFile.GetTargetPath(davItem, "", '/');
+ var pathUrl = DatabaseStoreSymlinkFile.GetTargetPath(davItem.Id, "", '/');
if (pathUrl.StartsWith('/')) pathUrl = pathUrl.TrimStart('/');
var strmKey = configManager.GetStrmKey();
var downloadKey = GetWebdavItemRequest.GenerateDownloadKey(strmKey, pathUrl);
- var extension = Path.GetExtension(davItem.Name).ToLowerInvariant().TrimStart('.');
+ var extension = Path.GetExtension(davItem.Name).ToLower().TrimStart('.');
return $"{baseUrl}/view/{pathUrl}?downloadKey={downloadKey}&extension={extension}";
}
}
\ No newline at end of file
diff --git a/backend/Tasks/StrmToSymlinksTask.cs b/backend/Tasks/StrmToSymlinksTask.cs
new file mode 100644
index 00000000..ac902f44
--- /dev/null
+++ b/backend/Tasks/StrmToSymlinksTask.cs
@@ -0,0 +1,106 @@
+using System.Web;
+using Microsoft.EntityFrameworkCore;
+using NzbWebDAV.Config;
+using NzbWebDAV.Database;
+using NzbWebDAV.Extensions;
+using NzbWebDAV.Utils;
+using NzbWebDAV.WebDav;
+using NzbWebDAV.Websocket;
+using Serilog;
+
+namespace NzbWebDAV.Tasks;
+
+public class StrmToSymlinksTask(
+ ConfigManager configManager,
+ DavDatabaseClient dbClient,
+ WebsocketManager websocketManager
+) : BaseTask
+{
+ protected override async Task ExecuteInternal()
+ {
+ try
+ {
+ var ct = SigtermUtil.GetCancellationToken();
+ await ConvertAllStrmFilesToSymlinks(ct);
+ }
+ catch (Exception e)
+ {
+ Report($"Failed: {e.Message}");
+ Log.Error(e, "Failed to convert *.strm files to symlinks.");
+ }
+ }
+
+ private async Task ConvertAllStrmFilesToSymlinks(CancellationToken token)
+ {
+ var completedCount = 0;
+ var debounce = DebounceUtil.CreateDebounce(TimeSpan.FromMilliseconds(200));
+ var batches = OrganizedLinksUtil.GetLibraryDavItemLinks(configManager)
+ .Where(x => x.SymlinkOrStrmInfo is SymlinkAndStrmUtil.StrmInfo)
+ .ToBatches(batchSize: 100);
+
+ ReportProgress("Scanning library for strm files...", completedCount);
+ foreach (var batch in batches)
+ await ConvertBatchOfStrmFilesToSymlinks(batch, OnItemCompleted, token);
+ ReportProgress("Done!", completedCount);
+ return;
+
+ void OnItemCompleted()
+ {
+ completedCount++;
+ debounce(() => ReportProgress("Scanning library for strm files...", completedCount));
+ }
+ }
+
+ private async Task ConvertBatchOfStrmFilesToSymlinks
+ (
+ List<OrganizedLinksUtil.DavItemLink> batch,
+ Action onItemCompleted,
+ CancellationToken token
+ )
+ {
+ var items = batch
+ .Select(x => new { Link = x, Extension = GetExtension(x) })
+ .ToList();
+ var davItemsToFetch = items
+ .Where(x => x.Extension is null)
+ .Select(x => x.Link.DavItemId)
+ .ToList();
+ var davItems = await dbClient.Ctx.Items
+ .Where(x => davItemsToFetch.Contains(x.Id))
+ .ToDictionaryAsync(x => x.Id, x => x, token);
+ var itemsWithExtension = items
+ .Select(x => new
+ {
+ Link = x.Link,
+ Extension = x.Extension ?? Path.GetExtension(davItems[x.Link.DavItemId].Name).TrimStart('.'),
+ })
+ .ToList();
+
+ var mountDir = configManager.GetRcloneMountDir();
+ foreach (var item in itemsWithExtension)
+ {
+ var symlinkPath = PathUtil.ReplaceExtension(item.Link.LinkPath, item.Extension);
+ var symlinkTarget = DatabaseStoreSymlinkFile.GetTargetPath(item.Link.DavItemId, mountDir);
+ File.CreateSymbolicLink(symlinkPath, symlinkTarget);
+ File.Delete(item.Link.LinkPath);
+ onItemCompleted?.Invoke();
+ }
+ }
+
+ private string? GetExtension(OrganizedLinksUtil.DavItemLink link)
+ {
+ if (link.SymlinkOrStrmInfo is not SymlinkAndStrmUtil.StrmInfo strmInfo) return null;
+ var queryParams = HttpUtility.ParseQueryString(new Uri(strmInfo.TargetUrl).Query);
+ return queryParams.Get("extension");
+ }
+
+ private void Report(string message)
+ {
+ _ = websocketManager.SendMessage(WebsocketTopic.StrmToSymlinksTaskProgress, message);
+ }
+
+ private void ReportProgress(string message, int completedCount)
+ {
+ Report($"{message}\nConverted: {completedCount} strm file(s) to symlinks.");
+ }
+}
\ No newline at end of file
diff --git a/backend/Utils/OrganizedLinksUtil.cs b/backend/Utils/OrganizedLinksUtil.cs
index 49b1770f..afdecb9a 100644
--- a/backend/Utils/OrganizedLinksUtil.cs
+++ b/backend/Utils/OrganizedLinksUtil.cs
@@ -108,7 +108,8 @@ public static class OrganizedLinksUtil
return new DavItemLink()
{
LinkPath = symlinkInfo.SymlinkPath,
- DavItemId = Guid.Parse(guid)
+ DavItemId = Guid.Parse(guid),
+ SymlinkOrStrmInfo = symlinkInfo
};
}
@@ -121,7 +122,8 @@ public static class OrganizedLinksUtil
return new DavItemLink()
{
LinkPath = strmInfo.StrmPath,
- DavItemId = Guid.Parse(guid)
+ DavItemId = Guid.Parse(guid),
+ SymlinkOrStrmInfo = strmInfo
};
}
@@ -129,5 +131,6 @@ public static class OrganizedLinksUtil
{
public string LinkPath; // Path to either a symlink or strm file.
public Guid DavItemId;
+ public SymlinkAndStrmUtil.ISymlinkOrStrmInfo SymlinkOrStrmInfo;
}
}
\ No newline at end of file
diff --git a/backend/Utils/PathUtil.cs b/backend/Utils/PathUtil.cs
index 0e922bb3..8f382906 100644
--- a/backend/Utils/PathUtil.cs
+++ b/backend/Utils/PathUtil.cs
@@ -9,4 +9,12 @@ public class PathUtil
? GetAllParentDirectories(directoryName).Prepend(directoryName)
: [];
}
+
+ public static string ReplaceExtension(string path, string newExtensions)
+ {
+ var directoryName = Path.GetDirectoryName(path);
+ var filenameWithoutExtension = Path.GetFileNameWithoutExtension(path);
+ var newFilename = $"{filenameWithoutExtension}.{newExtensions.TrimStart('.')}";
+ return Path.Join(directoryName, newFilename);
+ }
}
\ No newline at end of file
diff --git a/backend/WebDav/DatabaseStoreSymlinkFile.cs b/backend/WebDav/DatabaseStoreSymlinkFile.cs
index 6031cabb..00f7602b 100644
--- a/backend/WebDav/DatabaseStoreSymlinkFile.cs
+++ b/backend/WebDav/DatabaseStoreSymlinkFile.cs
@@ -1,6 +1,7 @@
using System.Text;
using NzbWebDAV.Config;
using NzbWebDAV.Database.Models;
+using NzbWebDAV.Extensions;
using NzbWebDAV.WebDav.Base;
namespace NzbWebDAV.WebDav;
@@ -21,16 +22,16 @@ public class DatabaseStoreSymlinkFile(DavItem davFile, ConfigManager configManag
private string GetTargetPath()
{
- return GetTargetPath(davFile, configManager.GetRcloneMountDir());
+ return GetTargetPath(davFile.Id, configManager.GetRcloneMountDir());
}
- public static string GetTargetPath(DavItem davFile, string mountDir, char? pathSeparator = null)
+ public static string GetTargetPath(Guid davItemId, string mountDir, char? pathSeparator = null)
{
- var pathParts = davFile.IdPrefix
+ var pathParts = davItemId.GetFiveLengthPrefix()
.Select(x => x.ToString())
.Prepend(DavItem.IdsFolder.Name)
.Prepend(mountDir)
- .Append(davFile.Id.ToString())
+ .Append(davItemId.ToString())
.ToArray();
return string.Join(pathSeparator ?? Path.DirectorySeparatorChar, pathParts);
}
diff --git a/backend/Websocket/WebsocketTopic.cs b/backend/Websocket/WebsocketTopic.cs
index 7554e376..a469b00f 100644
--- a/backend/Websocket/WebsocketTopic.cs
+++ b/backend/Websocket/WebsocketTopic.cs
@@ -6,6 +6,7 @@ public class WebsocketTopic
public static readonly WebsocketTopic UsenetConnections = new("cxs", TopicType.State);
public static readonly WebsocketTopic SymlinkTaskProgress = new("stp", TopicType.State);
public static readonly WebsocketTopic CleanupTaskProgress = new("ctp", TopicType.State);
+ public static readonly WebsocketTopic StrmToSymlinksTaskProgress = new("st2sy", TopicType.State);
public static readonly WebsocketTopic QueueItemStatus = new("qs", TopicType.State);
public static readonly WebsocketTopic QueueItemProgress = new("qp", TopicType.State);
public static readonly WebsocketTopic HealthItemStatus = new("hs", TopicType.State);