Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Updated MultiConnectionNntpClient to prioritize Body/Article commands.
nzbdav-dev282ac0124766
diff --git a/backend/Clients/Usenet/Concurrency/PrioritizedSemaphore.cs b/backend/Clients/Usenet/Concurrency/PrioritizedSemaphore.csnew file mode 100644index 00000000..0ac85079--- /dev/null+++ b/backend/Clients/Usenet/Concurrency/PrioritizedSemaphore.cs@@ -0,0 +1,145 @@+using UsenetSharp.Concurrency;++namespace NzbWebDAV.Clients.Usenet.Concurrency;++public class PrioritizedSemaphore : IDisposable+{+ private readonly LinkedList<TaskCompletionSource<bool>> _highPriorityWaiters = [];+ private readonly LinkedList<TaskCompletionSource<bool>> _lowPriorityWaiters = [];+ private readonly SemaphorePriorityOdds _priorityOdds;+ private int _currentCount;+ private bool _disposed = false;+ private readonly Lock _lock = new();+ private readonly Random _random = new();++ public PrioritizedSemaphore(int initialCount, SemaphorePriorityOdds? priorityOdds = null)+ {+ ArgumentOutOfRangeException.ThrowIfNegative(initialCount);+ _priorityOdds = priorityOdds ?? new SemaphorePriorityOdds { HighPriorityOdds = 1 };+ _currentCount = initialCount;+ }++ public Task WaitAsync(SemaphorePriority priority, CancellationToken cancellationToken = default)+ {+ lock (_lock)+ {+ if (_disposed)+ throw new ObjectDisposedException(nameof(AsyncSemaphore));++ if (_currentCount > 0)+ {+ _currentCount--;+ return Task.CompletedTask;+ }++ var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);+ var queue = priority == SemaphorePriority.High ? _highPriorityWaiters : _lowPriorityWaiters;+ var node = queue.AddLast(tcs);++ if (cancellationToken.CanBeCanceled)+ {+ var registration = cancellationToken.Register(() =>+ {+ var removed = false;+ lock (_lock)+ {+ try+ {+ queue.Remove(node);+ removed = true;+ }+ catch (InvalidOperationException)+ {+ // intentionally left blank+ }+ }++ if (removed)+ tcs.TrySetCanceled(cancellationToken);+ });++ tcs.Task.ContinueWith(_ => registration.Dispose(), TaskScheduler.Default);+ }++ return tcs.Task;+ }+ }++ public void Release()+ {+ TaskCompletionSource<bool>? toRelease = null;+ lock (_lock)+ {+ if (_disposed)+ throw new ObjectDisposedException(nameof(AsyncSemaphore));++ if (_highPriorityWaiters.Count == 0)+ {+ // if there are no high-priority waiters,+ // then release a low-priority waiter.+ toRelease = Release(_lowPriorityWaiters);+ }+ else if (_lowPriorityWaiters.Count == 0)+ {+ // if there are no low-priority waiters,+ // then release a high-priority waiter.+ toRelease = Release(_highPriorityWaiters);+ }+ else+ {+ // if there are both high-priority waiters and low-priority waiters,+ // then roll the dice to determine which to release, based on the given odds.+ var result = _random.NextDouble();+ var (one, two) = (_highPriorityWaiters, _lowPriorityWaiters);+ if (result > _priorityOdds.HighPriorityOdds)+ (one, two) = (two, one);+ toRelease = Release(one);+ if (toRelease == null)+ Release(two);+ }++ if (toRelease == null)+ {+ // if no waiters were ultimately released,+ // then increase the current count.+ _currentCount++;+ return;+ }+ }++ toRelease.TrySetResult(true);+ }++ private static TaskCompletionSource<bool>? Release(LinkedList<TaskCompletionSource<bool>> queue)+ {+ while (queue.Count > 0)+ {+ var node = queue.First!;+ queue.RemoveFirst();++ // Skip canceled tasks+ if (!node.Value.Task.IsCanceled)+ {+ return node.Value;+ }+ }++ return null;+ }++ public void Dispose()+ {+ List<TaskCompletionSource<bool>> waitersToCancel;+ lock (_lock)+ {+ if (_disposed) return;+ _disposed = true;+ waitersToCancel = _highPriorityWaiters.Concat(_lowPriorityWaiters).ToList();+ _highPriorityWaiters.Clear();+ _lowPriorityWaiters.Clear();+ }++ foreach (var tcs in waitersToCancel)+ tcs.TrySetException(new ObjectDisposedException(nameof(AsyncSemaphore)));+ }+}\ No newline at end of filediff --git a/backend/Clients/Usenet/Concurrency/SemaphorePriority.cs b/backend/Clients/Usenet/Concurrency/SemaphorePriority.csnew file mode 100644index 00000000..33cc4c05--- /dev/null+++ b/backend/Clients/Usenet/Concurrency/SemaphorePriority.cs@@ -0,0 +1,7 @@+namespace NzbWebDAV.Clients.Usenet.Concurrency;++public enum SemaphorePriority+{+ High,+ Low,+}\ No newline at end of filediff --git a/backend/Clients/Usenet/Concurrency/SemaphorePriorityOdds.cs b/backend/Clients/Usenet/Concurrency/SemaphorePriorityOdds.csnew file mode 100644index 00000000..47d8b1f3--- /dev/null+++ b/backend/Clients/Usenet/Concurrency/SemaphorePriorityOdds.cs@@ -0,0 +1,7 @@+namespace NzbWebDAV.Clients.Usenet.Concurrency;++public class SemaphorePriorityOdds+{+ public required double HighPriorityOdds { get; set; }+ public double LowPriorityOdds => 1.0 - HighPriorityOdds;+}\ No newline at end of filediff --git a/backend/Clients/Usenet/Connections/ConnectionPool.cs b/backend/Clients/Usenet/Connections/ConnectionPool.csindex 98301b78..62ba0f5e 100644--- a/backend/Clients/Usenet/Connections/ConnectionPool.cs+++ b/backend/Clients/Usenet/Connections/ConnectionPool.cs@@ -1,6 +1,6 @@using System.Collections.Concurrent;using System.Runtime.CompilerServices;-using UsenetSharp.Concurrency;+using NzbWebDAV.Clients.Usenet.Concurrency;namespace NzbWebDAV.Clients.Usenet.Connections;@@ -35,7 +35,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable/* --------------------------------- state --------------------------------------- */private readonly ConcurrentStack<Pooled> _idleConnections = new();- private readonly AsyncSemaphore _gate;+ private readonly PrioritizedSemaphore _gate;private readonly CancellationTokenSource _sweepCts = new();private readonly Task _sweeperTask; // keeps timer alive@@ -57,7 +57,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposableIdleTimeout = idleTimeout ?? TimeSpan.FromSeconds(30);_maxConnections = maxConnections;- _gate = new AsyncSemaphore(maxConnections);+ _gate = new PrioritizedSemaphore(maxConnections);_sweeperTask = Task.Run(SweepLoop); // background idle-reaper}@@ -68,14 +68,17 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable/// Waits until at least (`reservedCount` + 1) slots are free before acquiring one,/// ensuring that after acquisition at least `reservedCount` remain available./// </summary>- public async Task<ConnectionLock<T>> GetConnectionLockAsync(- CancellationToken cancellationToken = default)+ public async Task<ConnectionLock<T>> GetConnectionLockAsync+ (+ SemaphorePriority priority,+ CancellationToken cancellationToken = default+ ){// Make caller cancellation also cancel the wait on the gate.using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _sweepCts.Token);- await _gate.WaitAsync(linked.Token).ConfigureAwait(false);+ await _gate.WaitAsync(priority, linked.Token).ConfigureAwait(false);// Pool might have been disposed after wait returned:if (Volatile.Read(ref _disposed) == 1)diff --git a/backend/Clients/Usenet/INntpClient.cs b/backend/Clients/Usenet/INntpClient.csindex 49652c1f..1eeed97a 100644--- a/backend/Clients/Usenet/INntpClient.cs+++ b/backend/Clients/Usenet/INntpClient.cs@@ -36,9 +36,6 @@ public interface INntpClient : IDisposableTask<UsenetDateResponse> DateAsync(CancellationToken cancellationToken);- Task WaitForReadyAsync(- CancellationToken cancellationToken);-public async Task<UsenetYencHeader> GetYencHeadersAsync(string segmentId, CancellationToken ct){var decodedBodyResponse = await DecodedBodyAsync(segmentId, ct).ConfigureAwait(false);diff --git a/backend/Clients/Usenet/MultiConnectionNntpClient.cs b/backend/Clients/Usenet/MultiConnectionNntpClient.csindex c8e48f3b..a53d960b 100644--- a/backend/Clients/Usenet/MultiConnectionNntpClient.cs+++ b/backend/Clients/Usenet/MultiConnectionNntpClient.cs@@ -1,10 +1,21 @@-using NzbWebDAV.Clients.Usenet.Connections;+using NzbWebDAV.Clients.Usenet.Concurrency;+using NzbWebDAV.Clients.Usenet.Connections;using NzbWebDAV.Clients.Usenet.Models;using NzbWebDAV.Models;using UsenetSharp.Models;namespace NzbWebDAV.Clients.Usenet;+/// <summary>+/// This client is responsible for delegating NNTP commands to a connection pool.+/// * The connection pool enforces a maximum number of allowed connections+/// * When a connection is available, the NNTP command executes immediately+/// * When a connection is not available, the NNTP command waits until a connection becomes available.+/// * When multiple commands are awaiting a connection,+/// then BODY/ARTICLE commands have higher priority than STAT/HEAD/DATE commands.+/// </summary>+/// <param name="connectionPool"></param>+/// <param name="type"></param>public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPool, ProviderType type): INntpClient{@@ -26,19 +37,22 @@ public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPoopublic async Task<UsenetStatResponse> StatAsync(SegmentId segmentId, CancellationToken ct){- using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.Low;+ using var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.StatAsync(segmentId, ct).ConfigureAwait(false);}public async Task<UsenetHeadResponse> HeadAsync(SegmentId segmentId, CancellationToken ct){- using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.Low;+ using var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.HeadAsync(segmentId, ct).ConfigureAwait(false);}public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId, CancellationToken ct){- var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.High;+ var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.DecodedBodyAsync(segmentId, OnDone, ct).ConfigureAwait(false);void OnDone(ArticleBodyResult articleBodyResult)@@ -49,7 +63,8 @@ public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPoopublic async Task<UsenetDecodedArticleResponse> DecodedArticleAsync(SegmentId segmentId, CancellationToken ct){- var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.High;+ var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.DecodedArticleAsync(segmentId, OnDone, ct).ConfigureAwait(false);void OnDone(ArticleBodyResult articleBodyResult)@@ -60,15 +75,11 @@ public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPoopublic async Task<UsenetDateResponse> DateAsync(CancellationToken ct){- using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.Low;+ using var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.DateAsync(ct).ConfigureAwait(false);}- public async Task WaitForReadyAsync(CancellationToken ct)- {- using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);- }-public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId,@@ -76,7 +87,8 @@ public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPooCancellationToken ct){- var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.High;+ var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.DecodedBodyAsync(segmentId, OnDone, ct).ConfigureAwait(false);void OnDone(ArticleBodyResult articleBodyResult)@@ -93,7 +105,8 @@ public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPooCancellationToken ct){- var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ const SemaphorePriority priority = SemaphorePriority.High;+ var connectionLock = await connectionPool.GetConnectionLockAsync(priority, ct).ConfigureAwait(false);return await connectionLock.Connection.DecodedArticleAsync(segmentId, OnDone, ct).ConfigureAwait(false);void OnDone(ArticleBodyResult articleBodyResult)diff --git a/backend/Clients/Usenet/WrappingNntpClient.cs b/backend/Clients/Usenet/WrappingNntpClient.csindex ecaa4519..a51610b6 100644--- a/backend/Clients/Usenet/WrappingNntpClient.cs+++ b/backend/Clients/Usenet/WrappingNntpClient.cs@@ -43,11 +43,6 @@ public class WrappingNntpClient(INntpClient usenetClient) : INntpClientreturn _usenetClient.DateAsync(cancellationToken);}- public Task WaitForReadyAsync(CancellationToken cancellationToken)- {- return _usenetClient.WaitForReadyAsync(cancellationToken);- }-public Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId,Action<ArticleBodyResult>? onConnectionReadyAgain, CancellationToken cancellationToken){