Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Replace INntpClient implementation with UsenetSharp.
nzbdav-dev91078649efaf
diff --git a/backend/Clients/Usenet/BaseNntpClient.cs b/backend/Clients/Usenet/BaseNntpClient.csnew file mode 100644index 00000000..aba50334--- /dev/null+++ b/backend/Clients/Usenet/BaseNntpClient.cs@@ -0,0 +1,149 @@+using NzbWebDAV.Clients.Usenet.Models;+using NzbWebDAV.Exceptions;+using UsenetSharp.Clients;+using UsenetSharp.Models;+using UsenetSharp.Streams;++namespace NzbWebDAV.Clients.Usenet;++/// <summary>+/// This class has four responsibilities that differ from the underlying UsenetClient implementation+/// 1. throw `CouldNotConnectToUsenetException` after any connection error.+/// 2. throw `CouldNotLoginToUsenetException` after any login error.+/// 3. Provide yenc-decoded data for articles retrieved through article/body commands.+/// 4. throw `UsenetArticleNotFound` when articles do not exist, within article/body/head commands.+/// </summary>+public class BaseNntpClient : INntpClient+{+ private readonly UsenetClient _client = new();++ public async Task ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken)+ {+ try+ {+ await _client.ConnectAsync(host, port, useSsl, cancellationToken);+ }+ catch (Exception e)+ {+ const string message = "Could not connect to usenet host. Check connection settings.";+ throw new CouldNotConnectToUsenetException(message, e);+ }+ }++ public async Task<UsenetResponse> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken)+ {+ try+ {+ var response = await _client.AuthenticateAsync(user, pass, cancellationToken);+ if (!response.Success)+ {+ var message = $"Could not login to usenet host: {response.ResponseMessage}";+ throw new CouldNotLoginToUsenetException(message);+ }++ return response;+ }+ catch (Exception e)+ {+ throw new CouldNotLoginToUsenetException("Could not login to usenet host.", e);+ }+ }++ public Task<UsenetStatResponse> StatAsync(SegmentId segmentId, CancellationToken cancellationToken)+ {+ return _client.StatAsync(segmentId, cancellationToken);+ }++ public async Task<UsenetHeadResponse> HeadAsync(SegmentId segmentId, CancellationToken cancellationToken)+ {+ var headResponse = await _client.HeadAsync(segmentId, cancellationToken);++ if (headResponse.ResponseType != UsenetResponseType.ArticleRetrievedHeadFollows)+ throw new UsenetArticleNotFoundException(segmentId);++ return new UsenetHeadResponse()+ {+ SegmentId = headResponse.SegmentId,+ ResponseCode = headResponse.ResponseCode,+ ResponseMessage = headResponse.ResponseMessage,+ ArticleHeaders = headResponse.ArticleHeaders!+ };+ }++ public Task<UsenetDecodedBodyResponse> DecodedBodyAsync+ (+ SegmentId segmentId,+ CancellationToken cancellationToken+ )+ {+ return DecodedBodyAsync(segmentId, onConnectionReadyAgain: null, cancellationToken);+ }++ public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync+ (+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken cancellationToken+ )+ {+ var bodyResponse = await _client.BodyAsync(segmentId, onConnectionReadyAgain, cancellationToken);++ if (bodyResponse.ResponseType != UsenetResponseType.ArticleRetrievedBodyFollows)+ throw new UsenetArticleNotFoundException(segmentId);++ return new UsenetDecodedBodyResponse()+ {+ SegmentId = bodyResponse.SegmentId,+ ResponseCode = bodyResponse.ResponseCode,+ ResponseMessage = bodyResponse.ResponseMessage,+ Stream = new YencStream(bodyResponse.Stream!),+ };+ }++ public Task<UsenetDecodedArticleResponse> DecodedArticleAsync+ (+ SegmentId segmentId,+ CancellationToken cancellationToken+ )+ {+ return DecodedArticleAsync(segmentId, onConnectionReadyAgain: null, cancellationToken);+ }++ public async Task<UsenetDecodedArticleResponse> DecodedArticleAsync+ (+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken cancellationToken+ )+ {+ var articleResponse = await _client.ArticleAsync(segmentId, onConnectionReadyAgain, cancellationToken);++ if (articleResponse.ResponseType != UsenetResponseType.ArticleRetrievedHeadAndBodyFollow)+ throw new UsenetArticleNotFoundException(segmentId);++ return new UsenetDecodedArticleResponse()+ {+ SegmentId = articleResponse.SegmentId,+ ResponseCode = articleResponse.ResponseCode,+ ResponseMessage = articleResponse.ResponseMessage,+ ArticleHeaders = articleResponse.ArticleHeaders!,+ Stream = new YencStream(articleResponse.Stream!),+ };+ }++ public Task<UsenetDateResponse> DateAsync(CancellationToken cancellationToken)+ {+ return _client.DateAsync(cancellationToken);+ }++ public Task WaitForReadyAsync(CancellationToken cancellationToken)+ {+ return _client.WaitForReadyAsync(cancellationToken);+ }++ public void Dispose()+ {+ _client.Dispose();+ GC.SuppressFinalize(this);+ }+}\ No newline at end of filediff --git a/backend/Clients/Usenet/CachingNntpClient.cs b/backend/Clients/Usenet/CachingNntpClient.csdeleted file mode 100644index c05c990d..00000000--- a/backend/Clients/Usenet/CachingNntpClient.cs+++ /dev/null@@ -1,45 +0,0 @@-using Microsoft.Extensions.Caching.Memory;-using NzbWebDAV.Streams;-using Usenet.Nzb;-using Usenet.Yenc;--namespace NzbWebDAV.Clients.Usenet;--public class CachingNntpClient(INntpClient client, MemoryCache cache) : WrappingNntpClient(client)-{- private readonly MemoryCacheEntryOptions _cacheOptions = new()- {- Size = 1,- SlidingExpiration = TimeSpan.FromHours(3)- };-- public override async Task<YencHeaderStream> GetSegmentStreamAsync- (- string segmentId,- bool includeHeaders,- CancellationToken ct- )- {- var cacheKey = segmentId;- var stream = await Client.GetSegmentStreamAsync(segmentId, includeHeaders, ct).ConfigureAwait(false);- cache.Set(cacheKey, stream.Header, _cacheOptions);- return stream;- }-- public override async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken ct)- {- var cacheKey = segmentId;- return (await cache.GetOrCreateAsync(cacheKey, cacheEntry =>- {- cacheEntry.SetOptions(_cacheOptions);- return Client.GetSegmentYencHeaderAsync(segmentId, ct);- }).ConfigureAwait(false))!;- }-- public override async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken ct)- {- if (file.Segments.Count == 0) return 0;- var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId.Value, ct).ConfigureAwait(false);- return header.PartOffset + header.PartSize;- }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Connections/CombinedSemaphoreSlim.cs b/backend/Clients/Usenet/Connections/CombinedSemaphoreSlim.csdeleted file mode 100644index a02a7671..00000000--- a/backend/Clients/Usenet/Connections/CombinedSemaphoreSlim.cs+++ /dev/null@@ -1,34 +0,0 @@-namespace NzbWebDAV.Clients.Usenet.Connections-{- public sealed class CombinedSemaphoreSlim(int maxCount, ExtendedSemaphoreSlim pooledSemaphore): IDisposable- {- private readonly SemaphoreSlim _semaphore = new(maxCount, maxCount);-- public int RemainingSemaphoreSlots => _semaphore.CurrentCount;-- public async Task WaitAsync(int requiredAvailable, CancellationToken cancellationToken)- {- await _semaphore.WaitAsync(cancellationToken);- try- {- await pooledSemaphore.WaitAsync(requiredAvailable, cancellationToken);- }- catch (Exception)- {- _semaphore.Release();- throw;- }- }-- public void Release()- {- _semaphore.Release();- pooledSemaphore.Release();- }-- public void Dispose()- {- _semaphore.Dispose();- }- }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Connections/ConnectionPool.cs b/backend/Clients/Usenet/Connections/ConnectionPool.csindex c4145bc5..98301b78 100644--- a/backend/Clients/Usenet/Connections/ConnectionPool.cs+++ b/backend/Clients/Usenet/Connections/ConnectionPool.cs@@ -1,6 +1,6 @@using System.Collections.Concurrent;using System.Runtime.CompilerServices;-using NzbWebDAV.Extensions;+using UsenetSharp.Concurrency;namespace NzbWebDAV.Clients.Usenet.Connections;@@ -26,7 +26,6 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposablepublic int IdleConnections => _idleConnections.Count;public int ActiveConnections => _live - _idleConnections.Count;public int AvailableConnections => _maxConnections - ActiveConnections;- public int RemainingSemaphoreSlots => _gate.RemainingSemaphoreSlots;public event EventHandler<ConnectionPoolStats.ConnectionPoolChangedEventArgs>? OnConnectionPoolChanged;@@ -36,7 +35,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable/* --------------------------------- state --------------------------------------- */private readonly ConcurrentStack<Pooled> _idleConnections = new();- private readonly CombinedSemaphoreSlim _gate;+ private readonly AsyncSemaphore _gate;private readonly CancellationTokenSource _sweepCts = new();private readonly Task _sweeperTask; // keeps timer alive@@ -47,7 +46,6 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposablepublic ConnectionPool(int maxConnections,- ExtendedSemaphoreSlim pooledSemaphore,Func<CancellationToken, ValueTask<T>> connectionFactory,TimeSpan? idleTimeout = null){@@ -59,7 +57,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposableIdleTimeout = idleTimeout ?? TimeSpan.FromSeconds(30);_maxConnections = maxConnections;- _gate = new CombinedSemaphoreSlim(maxConnections, pooledSemaphore);+ _gate = new AsyncSemaphore(maxConnections);_sweeperTask = Task.Run(SweepLoop); // background idle-reaper}@@ -73,13 +71,11 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposablepublic async Task<ConnectionLock<T>> GetConnectionLockAsync(CancellationToken cancellationToken = default){- var reservedCount = cancellationToken.GetContext<ReservedPooledConnectionsContext>().Count;-// Make caller cancellation also cancel the wait on the gate.using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _sweepCts.Token);- await _gate.WaitAsync(reservedCount, linked.Token).ConfigureAwait(false);+ await _gate.WaitAsync(linked.Token).ConfigureAwait(false);// Pool might have been disposed after wait returned:if (Volatile.Read(ref _disposed) == 1)@@ -98,7 +94,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable}// Stale – destroy and continue looking.- await DisposeConnectionAsync(item.Connection).ConfigureAwait(false);+ DisposeConnection(item.Connection);Interlocked.Decrement(ref _live);TriggerConnectionPoolChangedEvent();}@@ -142,7 +138,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable{if (Volatile.Read(ref _disposed) == 1){- _ = DisposeConnectionAsync(connection); // fire & forget+ DisposeConnection(connection);Interlocked.Decrement(ref _live);TriggerConnectionPoolChangedEvent();return;@@ -156,7 +152,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposableprivate void Destroy(T connection){// When a lock requests replacement, we dispose the connection instead of reusing.- _ = DisposeConnectionAsync(connection); // fire & forget+ DisposeConnection(connection);Interlocked.Decrement(ref _live);if (Volatile.Read(ref _disposed) == 0){@@ -183,7 +179,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable{using var timer = new PeriodicTimer(IdleTimeout / 2);while (await timer.WaitForNextTickAsync(_sweepCts.Token).ConfigureAwait(false))- await SweepOnce().ConfigureAwait(false);+ SweepOnce();}catch (OperationCanceledException){@@ -191,7 +187,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable}}- private async Task SweepOnce()+ private void SweepOnce(){var now = Environment.TickCount64;var survivors = new List<Pooled>();@@ -201,7 +197,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable{if (item.IsExpired(IdleTimeout, now)){- await DisposeConnectionAsync(item.Connection).ConfigureAwait(false);+ DisposeConnection(item.Connection);Interlocked.Decrement(ref _live);isAnyConnectionFreed = true;}@@ -212,7 +208,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable}// Preserve original LIFO order.- for (int i = survivors.Count - 1; i >= 0; i--)+ for (var i = survivors.Count - 1; i >= 0; i--)_idleConnections.Push(survivors[i]);if (isAnyConnectionFreed)@@ -221,17 +217,10 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable/* ------------------------- dispose helpers ------------------------------------ */- private static async ValueTask DisposeConnectionAsync(T conn)+ private static void DisposeConnection(T conn){- switch (conn)- {- case IAsyncDisposable ad:- await ad.DisposeAsync().ConfigureAwait(false);- break;- case IDisposable d:- d.Dispose();- break;- }+ if (conn is IDisposable d)+ d.Dispose();}/* -------------------------- IAsyncDisposable ---------------------------------- */@@ -240,7 +229,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable{if (Interlocked.Exchange(ref _disposed, 1) == 1) return;- _sweepCts.Cancel();+ await _sweepCts.CancelAsync();try{@@ -253,7 +242,7 @@ public sealed class ConnectionPool<T> : IDisposable, IAsyncDisposable// Drain and dispose cached items.while (_idleConnections.TryPop(out var item))- await DisposeConnectionAsync(item.Connection).ConfigureAwait(false);+ DisposeConnection(item.Connection);_sweepCts.Dispose();_gate.Dispose();diff --git a/backend/Clients/Usenet/Connections/ExtendedSemaphoreSlim.cs b/backend/Clients/Usenet/Connections/ExtendedSemaphoreSlim.csdeleted file mode 100644index 0ea403ed..00000000--- a/backend/Clients/Usenet/Connections/ExtendedSemaphoreSlim.cs+++ /dev/null@@ -1,252 +0,0 @@-using NzbWebDAV.Utils;--namespace NzbWebDAV.Clients.Usenet.Connections-{- /// <summary>- /// A semaphore-like primitive where each waiter specifies the number of- /// free slots that must remain available after acquisition.- ///- /// WaitAsync(requiredAvailable): acquires only if (CurrentCount >= requiredAvailable + 1).- /// Release(): releases one slot.- ///- /// Higher requiredAvailable implies higher priority when multiple waiters exist.- /// </summary>- public sealed class ExtendedSemaphoreSlim- {- private readonly object _lock = new object();-- // Available slots (like SemaphoreSlim's CurrentCount)- private int _currentCount;- private readonly int _maxCount;-- // Waiters bucketed by requiredAvailable; keys sorted ascending, we serve descending.- private readonly SortedDictionary<int, LinkedList<Waiter>> _queues = new();- private int _waiterCount; // tracked atomically for fast-path guard-- public ExtendedSemaphoreSlim(int initialCount, int maxCount)- {- if (maxCount <= 0) throw new ArgumentOutOfRangeException(nameof(maxCount));- if (initialCount < 0 || initialCount > maxCount)- throw new ArgumentOutOfRangeException(nameof(initialCount));-- _currentCount = initialCount;- _maxCount = maxCount;- }-- public ExtendedSemaphoreSlim(int initialCount) : this(initialCount, int.MaxValue)- {- }-- /// <summary>- /// Current number of available slots.- /// </summary>- public int CurrentCount => Volatile.Read(ref _currentCount);-- /// <summary>- /// Wait until there are at least (requiredAvailable + 1) slots free, then acquire 1 slot.- /// After this completes successfully, at least requiredAvailable slots remain free.- /// </summary>- public Task WaitAsync(int requiredAvailable, CancellationToken cancellationToken = default)- {- if (requiredAvailable < 0) throw new ArgumentOutOfRangeException(nameof(requiredAvailable));- if (requiredAvailable >= _maxCount)- throw new ArgumentOutOfRangeException(nameof(requiredAvailable),- "requiredAvailable must be less than the semaphore's maximum count.");-- // Fast path: no waiters and enough capacity -> try to take one with a single CAS.- if (Volatile.Read(ref _waiterCount) == 0 && TryAcquireFast(requiredAvailable))- return Task.CompletedTask;-- // Slow path: enqueue and let the scheduler (by priority) grant slots.- var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);- var waiter = new Waiter(requiredAvailable, tcs);-- List<Waiter>? toGrant;- CancellationTokenRegistration ctr = default;-- lock (_lock)- {- EnqueueWaiter_NoLock(waiter);-- // Try to grant immediately in case capacity is there (or arrived just before we enqueued).- toGrant = TryGrantWaiters_NoLock();- }-- // Fire completions outside the lock.- CompleteGranted(toGrant);-- if (tcs.Task.IsCompleted)- return tcs.Task;-- if (cancellationToken.CanBeCanceled)- {- ctr = cancellationToken.Register(static state =>- {- var (sem, w, tok) = ((ExtendedSemaphoreSlim, Waiter, CancellationToken))state!;- bool removed = sem.TryRemoveWaiter(w);- if (removed)- {- w.Tcs.TrySetCanceled(tok);- }- }, (this, waiter, cancellationToken));-- // Dispose registration when the wait completes in any way.- _ = tcs.Task.ContinueWith(static (_, s) => { ((CancellationTokenRegistration)s!).Dispose(); }, ctr,- SigtermUtil.GetCancellationToken(),- TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);- }-- return tcs.Task;-- // ---------------- local helpers ------------------ bool TryAcquireFast(int reqAvail)- {- while (true)- {- int observed = Volatile.Read(ref _currentCount);- if (observed <= reqAvail) return false; // need at least reqAvail+1 to take one- int newVal = observed - 1;- if (Interlocked.CompareExchange(ref _currentCount, newVal, observed) == observed)- return true;- }- }- }-- /// <summary>- /// Release one slot back to the semaphore.- /// </summary>- public void Release()- {- int newCount = Interlocked.Increment(ref _currentCount);- if (newCount > _maxCount)- {- Interlocked.Decrement(ref _currentCount);- throw new SemaphoreFullException();- }-- // If there are no waiters, we're done.- if (Volatile.Read(ref _waiterCount) == 0)- return;-- List<Waiter>? toGrant;- lock (_lock)- {- toGrant = TryGrantWaiters_NoLock();- }-- // Complete outside the lock to avoid running continuations under the lock.- CompleteGranted(toGrant);- }-- // ---------------- core scheduling ------------------ private void EnqueueWaiter_NoLock(Waiter w)- {- if (!_queues.TryGetValue(w.Required, out var list))- {- list = new LinkedList<Waiter>();- _queues.Add(w.Required, list);- }-- w.Node = list.AddLast(w);- Interlocked.Increment(ref _waiterCount);- }-- private bool TryRemoveWaiter(Waiter w)- {- lock (_lock)- {- if (w.Signaled) return false; // already granted- if (w.Node == null) return false; // not in a queue (already removed)- if (!_queues.TryGetValue(w.Required, out var list)) return false;-- list.Remove(w.Node);- w.Node = null;- Interlocked.Decrement(ref _waiterCount);- if (list.Count == 0) _queues.Remove(w.Required);- return true;- }- }-- /// <summary>- /// Grants as many queued waiters as possible by descending requiredAvailable.- /// Must be called under _lock. Returns the waiters to complete outside the lock.- /// </summary>- private List<Waiter> TryGrantWaiters_NoLock()- {- if (Volatile.Read(ref _waiterCount) == 0)- return s_empty;-- var granted = new List<Waiter>();- List<int>? emptyKeys = null;-- // Iterate priorities descending; serve FIFO within each priority.- foreach (var kvp in _queues.Reverse())- {- int req = kvp.Key;- var list = kvp.Value;-- while (list.First is not null)- {- int observed = Volatile.Read(ref _currentCount);- if (observed <= req)- break;-- int newCount = Interlocked.Decrement(ref _currentCount);- if (newCount < 0)- {- Interlocked.Increment(ref _currentCount);- throw new InvalidOperationException("Semaphore count went negative.");- }-- var node = list.First!;- list.RemoveFirst();-- var w = node.Value;- w.Node = null;- w.Signaled = true;-- Interlocked.Decrement(ref _waiterCount);- granted.Add(w);- }-- if (list.Count == 0)- (emptyKeys ??= new List<int>()).Add(req);- }-- if (emptyKeys is not null)- {- foreach (var k in emptyKeys)- _queues.Remove(k);- }-- return granted;- }-- private static void CompleteGranted(List<Waiter>? granted)- {- if (granted is null || granted.Count == 0) return;- foreach (var w in granted)- w.Tcs.TrySetResult(true);- }-- // ---------------- inner types ------------------ private sealed class Waiter- {- public readonly int Required;- public readonly TaskCompletionSource<bool> Tcs;- public LinkedListNode<Waiter>? Node;- public bool Signaled;-- public Waiter(int required, TaskCompletionSource<bool> tcs)- {- Required = required;- Tcs = tcs;- }- }-- private static readonly List<Waiter> s_empty = new List<Waiter>(0);- }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Connections/LastSuccessfulProviderContext.cs b/backend/Clients/Usenet/Connections/LastSuccessfulProviderContext.csdeleted file mode 100644index 75566629..00000000--- a/backend/Clients/Usenet/Connections/LastSuccessfulProviderContext.cs+++ /dev/null@@ -1,6 +0,0 @@-namespace NzbWebDAV.Clients.Usenet.Connections;--public class LastSuccessfulProviderContext-{- public MultiConnectionNntpClient? Provider;-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Connections/ReservedPooledConnectionsContext.cs b/backend/Clients/Usenet/Connections/ReservedPooledConnectionsContext.csdeleted file mode 100644index 83c04a7d..00000000--- a/backend/Clients/Usenet/Connections/ReservedPooledConnectionsContext.cs+++ /dev/null@@ -1,6 +0,0 @@-namespace NzbWebDAV.Clients.Usenet.Connections;--public readonly struct ReservedPooledConnectionsContext(int reservedCount)-{- public readonly int Count = Math.Max(reservedCount, 0);-}\ No newline at end of filediff --git a/backend/Clients/Usenet/INntpClient.cs b/backend/Clients/Usenet/INntpClient.csindex 6b938676..49652c1f 100644--- a/backend/Clients/Usenet/INntpClient.cs+++ b/backend/Clients/Usenet/INntpClient.cs@@ -1,20 +1,101 @@using NzbWebDAV.Clients.Usenet.Models;+using NzbWebDAV.Exceptions;+using NzbWebDAV.Extensions;using NzbWebDAV.Streams;-using Usenet.Nntp.Responses;using Usenet.Nzb;-using Usenet.Yenc;+using UsenetSharp.Models;namespace NzbWebDAV.Clients.Usenet;-public interface INntpClient: IDisposable+public interface INntpClient : IDisposable{- Task<bool> ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken);- Task<bool> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken);- Task<NntpStatResponse> StatAsync(string segmentId, CancellationToken cancellationToken);- Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, bool includeHeaders, CancellationToken ct);- Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken);- Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken);- Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken);- Task<NntpDateResponse> DateAsync(CancellationToken cancellationToken);- Task WaitForReady(CancellationToken cancellationToken);+ Task ConnectAsync(+ string host, int port, bool useSsl, CancellationToken cancellationToken);++ Task<UsenetResponse> AuthenticateAsync(+ string user, string pass, CancellationToken cancellationToken);++ Task<UsenetStatResponse> StatAsync(+ SegmentId segmentId, CancellationToken cancellationToken);++ Task<UsenetHeadResponse> HeadAsync(+ SegmentId segmentId, CancellationToken cancellationToken);++ Task<UsenetDecodedBodyResponse> DecodedBodyAsync(+ SegmentId segmentId, CancellationToken cancellationToken);++ Task<UsenetDecodedBodyResponse> DecodedBodyAsync(+ SegmentId segmentId, Action<ArticleBodyResult>? onConnectionReadyAgain, CancellationToken cancellationToken);++ Task<UsenetDecodedArticleResponse> DecodedArticleAsync(+ SegmentId segmentId, CancellationToken cancellationToken);++ Task<UsenetDecodedArticleResponse> DecodedArticleAsync(+ SegmentId segmentId, Action<ArticleBodyResult>? onConnectionReadyAgain, CancellationToken cancellationToken);++ Task<UsenetDateResponse> DateAsync(+ CancellationToken cancellationToken);++ Task WaitForReadyAsync(+ CancellationToken cancellationToken);++ public async Task<UsenetYencHeader> GetYencHeadersAsync(string segmentId, CancellationToken ct)+ {+ var decodedBodyResponse = await DecodedBodyAsync(segmentId, ct).ConfigureAwait(false);+ await using var stream = decodedBodyResponse.Stream;+ var headers = await stream.GetYencHeadersAsync(ct).ConfigureAwait(false);+ return headers!;+ }++ public async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken ct)+ {+ if (file.Segments.Count == 0) return 0;+ var headers = await GetYencHeadersAsync(file.Segments[^1].MessageId.Value, ct).ConfigureAwait(false);+ return headers!.PartOffset + headers!.PartSize;+ }++ public async Task<NzbFileStream> GetFileStream(NzbFile nzbFile, int concurrentConnections, CancellationToken ct)+ {+ var segmentIds = nzbFile.GetSegmentIds();+ var fileSize = await GetFileSizeAsync(nzbFile, ct).ConfigureAwait(false);+ return new NzbFileStream(segmentIds, fileSize, this, concurrentConnections);+ }++ public NzbFileStream GetFileStream(NzbFile nzbFile, long fileSize, int concurrentConnections)+ {+ return new NzbFileStream(nzbFile.GetSegmentIds(), fileSize, this, concurrentConnections);+ }++ public NzbFileStream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections)+ {+ return new NzbFileStream(segmentIds, fileSize, this, concurrentConnections);+ }++ public async Task CheckAllSegmentsAsync+ (+ IEnumerable<string> segmentIds,+ int concurrency,+ IProgress<int>? progress,+ CancellationToken cancellationToken+ )+ {+ using var childCt = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);+ var token = childCt.Token;++ var tasks = segmentIds+ .Select(async segmentId => (+ SegmentId: segmentId,+ Result: await StatAsync(segmentId, token).ConfigureAwait(false)+ ))+ .WithConcurrencyAsync(concurrency);++ var processed = 0;+ await foreach (var task in tasks.ConfigureAwait(false))+ {+ progress?.Report(++processed);+ if (task.Result.ResponseType == UsenetResponseType.ArticleExists) continue;+ await childCt.CancelAsync().ConfigureAwait(false);+ throw new UsenetArticleNotFoundException(task.SegmentId);+ }+ }}\ No newline at end of filediff --git a/backend/Clients/Usenet/Models/UsenetArticle.cs b/backend/Clients/Usenet/Models/UsenetArticle.csdeleted file mode 100644index 9246484f..00000000--- a/backend/Clients/Usenet/Models/UsenetArticle.cs+++ /dev/null@@ -1,7 +0,0 @@-namespace NzbWebDAV.Clients.Usenet.Models;--public class UsenetArticle-{- public UsenetArticleHeaders? Headers { get; init; }- public required IEnumerable<string> Body { get; init; }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Models/UsenetArticleHeaders.cs b/backend/Clients/Usenet/Models/UsenetArticleHeaders.csdeleted file mode 100644index 8fa29e41..00000000--- a/backend/Clients/Usenet/Models/UsenetArticleHeaders.cs+++ /dev/null@@ -1,134 +0,0 @@-using System.Collections.Immutable;-using System.Globalization;-using System.Text.RegularExpressions;--namespace NzbWebDAV.Clients.Usenet.Models;--public class UsenetArticleHeaders-{- public DateTimeOffset Date { get; init; }- public ImmutableHashSet<string> NewsGroups { get; init; }- public ImmutableHashSet<string> Organization { get; init; }- public ImmutableHashSet<string> Subject { get; init; }- public ImmutableHashSet<string> MessageId { get; init; }- public ImmutableHashSet<string> From { get; init; }-- public UsenetArticleHeaders(ImmutableDictionary<string, ImmutableHashSet<string>> headers)- {- Date = GetDate(headers.GetValueOrDefault("Date", []));- NewsGroups = headers.GetValueOrDefault("Newsgroups", []);- Organization = headers.GetValueOrDefault("Organization", []);- Subject = headers.GetValueOrDefault("Subject", []);- MessageId = headers.GetValueOrDefault("Message-Id", []);- From = headers.GetValueOrDefault("From", []);- }-- private static DateTimeOffset GetDate(ImmutableHashSet<string> date)- {- return !date.IsEmpty && TryParseDateUtc(date.First(), out var parsedDate)- ? parsedDate- : DateTimeOffset.UtcNow;- }-- // Comprehensive zone map (RFC 822 + common real-world + military)- private static readonly Dictionary<string, string> TimeZoneOffsets = new(StringComparer.OrdinalIgnoreCase)- {- // RFC 822- ["UT"] = "+0000", ["UTC"] = "+0000", ["GMT"] = "+0000",- ["EST"] = "-0500", ["EDT"] = "-0400",- ["CST"] = "-0600", ["CDT"] = "-0500",- ["MST"] = "-0700", ["MDT"] = "-0600",- ["PST"] = "-0800", ["PDT"] = "-0700",-- // Common European- ["BST"] = "+0100", ["CET"] = "+0100", ["CEST"] = "+0200",- ["EET"] = "+0200", ["EEST"] = "+0300",- ["WET"] = "+0000", ["WEST"] = "+0100",-- // Asia–Pacific (often seen in NNTP)- ["IST"] = "+0530", ["JST"] = "+0900", ["KST"] = "+0900",- ["AEST"] = "+1000", ["AEDT"] = "+1100",- ["ACST"] = "+0930", ["ACDT"] = "+1030",- ["AWST"] = "+0800",- ["NZST"] = "+1200", ["NZDT"] = "+1300",-- // Military letters (A–Z except J)- ["A"] = "+0100", ["B"] = "+0200", ["C"] = "+0300", ["D"] = "+0400",- ["E"] = "+0500", ["F"] = "+0600", ["G"] = "+0700", ["H"] = "+0800",- ["I"] = "+0900", ["K"] = "+1000", ["L"] = "+1100", ["M"] = "+1200",- ["N"] = "-0100", ["O"] = "-0200", ["P"] = "-0300", ["Q"] = "-0400",- ["R"] = "-0500", ["S"] = "-0600", ["T"] = "-0700", ["U"] = "-0800",- ["V"] = "-0900", ["W"] = "-1000", ["X"] = "-1100", ["Y"] = "-1200",- ["Z"] = "+0000"- };-- private static readonly string[] DateFormats =- {- // RFC 5322-style (preferred)- "ddd, dd MMM yyyy HH:mm:ss zzz",- "ddd, dd MMM yyyy HH:mm zzz",- // Legacy 2-digit year- "ddd, dd MMM yy HH:mm:ss zzz",- "ddd, dd MMM yy HH:mm zzz",- // Missing comma variations- "dd MMM yyyy HH:mm:ss zzz",- "dd MMM yy HH:mm:ss zzz",- // Occasionally missing seconds- "ddd, dd MMM yyyy HH:mm zzz",- "ddd, dd MMM yy HH:mm zzz"- };-- private static bool TryParseDateUtc(string input, out DateTimeOffset dateTimeOffset)- {- dateTimeOffset = default;- if (string.IsNullOrWhiteSpace(input))- return false;-- // Remove comments like "(UTC)"- string cleaned = Regex.Replace(input, @"\s*\(.*?\)\s*", " ");-- // Collapse multiple spaces- cleaned = Regex.Replace(cleaned.Trim(), @"\s+", " ");-- // Normalize known named time zones- foreach (var kvp in TimeZoneOffsets)- {- cleaned = Regex.Replace(- cleaned,- $@"\b{Regex.Escape(kvp.Key)}\b",- kvp.Value,- RegexOptions.IgnoreCase);- }-- // Ensure we have something like +0000 at the end if possible- cleaned = Regex.Replace(cleaned, @"\s([0-9]{4})$", " +$1");-- // Try each explicit format- foreach (var fmt in DateFormats)- {- if (DateTimeOffset.TryParseExact(- cleaned,- fmt,- CultureInfo.InvariantCulture,- DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal,- out dateTimeOffset))- {- dateTimeOffset = dateTimeOffset.ToUniversalTime();- return true;- }- }-- // Fallback flexible parse- if (DateTimeOffset.TryParse(- cleaned,- CultureInfo.InvariantCulture,- DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal,- out dateTimeOffset))- {- dateTimeOffset = dateTimeOffset.ToUniversalTime();- return true;- }-- return false;- }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/Models/UsenetDecodedArticleResponse.cs b/backend/Clients/Usenet/Models/UsenetDecodedArticleResponse.csnew file mode 100644index 00000000..21a053fd--- /dev/null+++ b/backend/Clients/Usenet/Models/UsenetDecodedArticleResponse.cs@@ -0,0 +1,11 @@+using UsenetSharp.Models;+using UsenetSharp.Streams;++namespace NzbWebDAV.Clients.Usenet.Models;++public record UsenetDecodedArticleResponse : UsenetResponse+{+ public required string SegmentId { get; init; }+ public required YencStream Stream { get; init; }+ public required UsenetArticleHeader ArticleHeaders { get; init; }+}\ No newline at end of filediff --git a/backend/Clients/Usenet/Models/UsenetDecodedBodyResponse.cs b/backend/Clients/Usenet/Models/UsenetDecodedBodyResponse.csnew file mode 100644index 00000000..4e87b85a--- /dev/null+++ b/backend/Clients/Usenet/Models/UsenetDecodedBodyResponse.cs@@ -0,0 +1,10 @@+using UsenetSharp.Models;+using UsenetSharp.Streams;++namespace NzbWebDAV.Clients.Usenet.Models;++public record UsenetDecodedBodyResponse : UsenetResponse+{+ public required string SegmentId { get; init; }+ public required YencStream Stream { get; init; }+}\ No newline at end of filediff --git a/backend/Clients/Usenet/MultiConnectionNntpClient.cs b/backend/Clients/Usenet/MultiConnectionNntpClient.csindex fb438bb1..c8e48f3b 100644--- a/backend/Clients/Usenet/MultiConnectionNntpClient.cs+++ b/backend/Clients/Usenet/MultiConnectionNntpClient.cs@@ -1,126 +1,119 @@using NzbWebDAV.Clients.Usenet.Connections;using NzbWebDAV.Clients.Usenet.Models;using NzbWebDAV.Models;-using NzbWebDAV.Streams;-using NzbWebDAV.Utils;-using Usenet.Exceptions;-using Usenet.Nntp.Responses;-using Usenet.Nzb;-using Usenet.Yenc;+using UsenetSharp.Models;namespace NzbWebDAV.Clients.Usenet;-public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPool, ProviderType type) : INntpClient+public class MultiConnectionNntpClient(ConnectionPool<INntpClient> connectionPool, ProviderType type)+ : INntpClient{public ProviderType ProviderType { get; } = type;- public int LiveConnections => _connectionPool.LiveConnections;- public int IdleConnections => _connectionPool.IdleConnections;- public int ActiveConnections => _connectionPool.ActiveConnections;- public int AvailableConnections => _connectionPool.AvailableConnections;- public int RemainingSemaphoreSlots => _connectionPool.RemainingSemaphoreSlots;+ public int LiveConnections => connectionPool.LiveConnections;+ public int IdleConnections => connectionPool.IdleConnections;+ public int ActiveConnections => connectionPool.ActiveConnections;+ public int AvailableConnections => connectionPool.AvailableConnections;- private ConnectionPool<INntpClient> _connectionPool = connectionPool;-- public Task<bool> ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken)+ public Task ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken){throw new NotSupportedException("Please connect within the connectionFactory");}- public Task<bool> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken)+ public Task<UsenetResponse> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken){throw new NotSupportedException("Please authenticate within the connectionFactory");}- public Task<NntpStatResponse> StatAsync(string segmentId, CancellationToken cancellationToken)+ public async Task<UsenetStatResponse> StatAsync(SegmentId segmentId, CancellationToken ct){- return RunWithConnection(connection => connection.StatAsync(segmentId, cancellationToken), cancellationToken);+ using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.StatAsync(segmentId, ct).ConfigureAwait(false);}- public Task<NntpDateResponse> DateAsync(CancellationToken cancellationToken)+ public async Task<UsenetHeadResponse> HeadAsync(SegmentId segmentId, CancellationToken ct){- return RunWithConnection(connection => connection.DateAsync(cancellationToken), cancellationToken);+ using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.HeadAsync(segmentId, ct).ConfigureAwait(false);}- public Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken)+ public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId, CancellationToken ct){- return RunWithConnection(connection => connection.GetArticleHeadersAsync(segmentId, cancellationToken),- cancellationToken);- }+ var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.DecodedBodyAsync(segmentId, OnDone, ct).ConfigureAwait(false);- public Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, bool includeHeaders,- CancellationToken cancellationToken)- {- return RunWithConnection(- connection => connection.GetSegmentStreamAsync(segmentId, includeHeaders, cancellationToken),- cancellationToken);+ void OnDone(ArticleBodyResult articleBodyResult)+ {+ connectionLock.Dispose();+ }}- public Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)+ public async Task<UsenetDecodedArticleResponse> DecodedArticleAsync(SegmentId segmentId, CancellationToken ct){- return RunWithConnection(connection => connection.GetSegmentYencHeaderAsync(segmentId, cancellationToken),- cancellationToken);+ var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.DecodedArticleAsync(segmentId, OnDone, ct).ConfigureAwait(false);++ void OnDone(ArticleBodyResult articleBodyResult)+ {+ connectionLock.Dispose();+ }}- public Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)+ public async Task<UsenetDateResponse> DateAsync(CancellationToken ct){- return RunWithConnection(connection => connection.GetFileSizeAsync(file, cancellationToken), cancellationToken);+ using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.DateAsync(ct).ConfigureAwait(false);}- public async Task WaitForReady(CancellationToken cancellationToken)+ public async Task WaitForReadyAsync(CancellationToken ct){- using var connectionLock =- await _connectionPool.GetConnectionLockAsync(cancellationToken).ConfigureAwait(false);+ using var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);}- private async Task<T> RunWithConnection<T>+ public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync(- Func<INntpClient, Task<T>> task,- CancellationToken cancellationToken,- int retries = 1+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken ct){- var connectionLock = await _connectionPool.GetConnectionLockAsync(cancellationToken).ConfigureAwait(false);- var isDisposed = false;- try- {- return await task(connectionLock.Connection).ConfigureAwait(false);- }- catch (NntpException)+ var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.DecodedBodyAsync(segmentId, OnDone, ct).ConfigureAwait(false);++ void OnDone(ArticleBodyResult articleBodyResult){- // we want to replace the underlying connection in cases of NntpExceptions.- connectionLock.Replace();connectionLock.Dispose();- isDisposed = true;+ onConnectionReadyAgain?.Invoke(articleBodyResult);+ }+ }- // and try again with a new connection (max 1 retry)- if (retries > 0)- return await RunWithConnection<T>(task, cancellationToken, retries - 1).ConfigureAwait(false);+ public async Task<UsenetDecodedArticleResponse> DecodedArticleAsync+ (+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken ct+ )+ {+ var connectionLock = await connectionPool.GetConnectionLockAsync(ct).ConfigureAwait(false);+ return await connectionLock.Connection.DecodedArticleAsync(segmentId, OnDone, ct).ConfigureAwait(false);- throw;- }- finally+ void OnDone(ArticleBodyResult articleBodyResult){- // we only want to release the connection-lock once the underlying connection is ready again.- //- // ReSharper disable once MethodSupportsCancellation- // we intentionally do not pass the cancellation token to ContinueWith,- // since we want the continuation to always run.- if (!isDisposed)- _ = connectionLock.Connection.WaitForReady(SigtermUtil.GetCancellationToken())- .ContinueWith(_ => connectionLock.Dispose());+ connectionLock.Dispose();+ onConnectionReadyAgain?.Invoke(articleBodyResult);}}- public void UpdateConnectionPool(ConnectionPool<INntpClient> connectionPool)+ public async Task<UsenetYencHeader> GetYencHeadersAsync(string segmentId, CancellationToken ct){- var oldConnectionPool = _connectionPool;- _connectionPool = connectionPool;- oldConnectionPool.Dispose();+ var decodedBodyResponse = await DecodedBodyAsync(segmentId, ct).ConfigureAwait(false);+ await using var stream = decodedBodyResponse.Stream;+ var headers = await stream.GetYencHeadersAsync(ct).ConfigureAwait(false);+ return headers!;}public void Dispose(){- _connectionPool.Dispose();+ connectionPool.Dispose();GC.SuppressFinalize(this);}}\ No newline at end of filediff --git a/backend/Clients/Usenet/MultiProviderNntpClient.cs b/backend/Clients/Usenet/MultiProviderNntpClient.csindex d676f43e..8667093c 100644--- a/backend/Clients/Usenet/MultiProviderNntpClient.cs+++ b/backend/Clients/Usenet/MultiProviderNntpClient.cs@@ -1,87 +1,138 @@using System.Runtime.ExceptionServices;-using NzbWebDAV.Clients.Usenet.Connections;using NzbWebDAV.Clients.Usenet.Models;-using NzbWebDAV.Exceptions;-using NzbWebDAV.Extensions;using NzbWebDAV.Models;-using NzbWebDAV.Streams;using Serilog;-using Usenet.Nntp.Responses;-using Usenet.Nzb;-using Usenet.Yenc;+using UsenetSharp.Models;namespace NzbWebDAV.Clients.Usenet;public class MultiProviderNntpClient(List<MultiConnectionNntpClient> providers) : INntpClient{- public Task<bool> ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken)+ public Task ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken){throw new NotSupportedException("Please connect within the connectionFactory");}- public Task<bool> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken)+ public Task<UsenetResponse> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken){throw new NotSupportedException("Please authenticate within the connectionFactory");}- public Task<NntpStatResponse> StatAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetStatResponse> StatAsync(SegmentId segmentId, CancellationToken cancellationToken){- return RunFromPoolWithBackup(connection => connection.StatAsync(segmentId, cancellationToken),- cancellationToken);+ return RunFromPoolWithBackup(x => x.StatAsync(segmentId, cancellationToken), cancellationToken);}- public Task<NntpDateResponse> DateAsync(CancellationToken cancellationToken)+ public Task<UsenetHeadResponse> HeadAsync(SegmentId segmentId, CancellationToken cancellationToken){- return RunFromPoolWithBackup(connection => connection.DateAsync(cancellationToken), cancellationToken);+ return RunFromPoolWithBackup(x => x.HeadAsync(segmentId, cancellationToken), cancellationToken);}- public Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId, CancellationToken cancellationToken){- return RunFromPoolWithBackup(connection => connection.GetArticleHeadersAsync(segmentId, cancellationToken),- cancellationToken);+ return RunFromPoolWithBackup(x => x.DecodedBodyAsync(segmentId, cancellationToken), cancellationToken);}- public Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, bool includeHeaders,- CancellationToken cancellationToken)+ public Task<UsenetDecodedArticleResponse> DecodedArticleAsync+ (+ SegmentId segmentId,+ CancellationToken cancellationToken+ ){- return RunFromPoolWithBackup(- connection => connection.GetSegmentStreamAsync(segmentId, includeHeaders, cancellationToken),- cancellationToken);+ return RunFromPoolWithBackup(x => x.DecodedArticleAsync(segmentId, cancellationToken), cancellationToken);}- public Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetDateResponse> DateAsync(CancellationToken cancellationToken){- return RunFromPoolWithBackup(connection => connection.GetSegmentYencHeaderAsync(segmentId, cancellationToken),- cancellationToken);+ return RunFromPoolWithBackup(x => x.DateAsync(cancellationToken), cancellationToken);}- public Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)+ public async Task<UsenetDecodedBodyResponse> DecodedBodyAsync+ (+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken cancellationToken+ ){- return RunFromPoolWithBackup(connection => connection.GetFileSizeAsync(file, cancellationToken),- cancellationToken);+ UsenetDecodedBodyResponse? result;+ try+ {+ result = await RunFromPoolWithBackup(+ x => x.DecodedBodyAsync(segmentId, OnConnectionReadyAgain, cancellationToken),+ cancellationToken+ ).ConfigureAwait(false);+ }+ catch+ {+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.NotRetrieved);+ throw;+ }++ if (result.ResponseType != UsenetResponseType.ArticleRetrievedBodyFollows)+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.NotRetrieved);++ return result;++ void OnConnectionReadyAgain(ArticleBodyResult articleBodyResult)+ {+ if (articleBodyResult == ArticleBodyResult.Retrieved)+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.Retrieved);+ }}- public Task WaitForReady(CancellationToken cancellationToken)+ public async Task<UsenetDecodedArticleResponse> DecodedArticleAsync+ (+ SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain,+ CancellationToken cancellationToken+ )+ {+ UsenetDecodedArticleResponse? result;+ try+ {+ result = await RunFromPoolWithBackup(+ x => x.DecodedArticleAsync(segmentId, OnConnectionReadyAgain, cancellationToken),+ cancellationToken+ ).ConfigureAwait(false);+ }+ catch+ {+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.NotRetrieved);+ throw;+ }++ if (result.ResponseType != UsenetResponseType.ArticleRetrievedHeadAndBodyFollow)+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.NotRetrieved);++ return result;++ void OnConnectionReadyAgain(ArticleBodyResult articleBodyResult)+ {+ if (articleBodyResult == ArticleBodyResult.Retrieved)+ onConnectionReadyAgain?.Invoke(ArticleBodyResult.Retrieved);+ }+ }++ public Task WaitForReadyAsync(CancellationToken cancellationToken){- return Task.CompletedTask;+ throw new NotSupportedException();}private async Task<T> RunFromPoolWithBackup<T>(Func<INntpClient, Task<T>> task,CancellationToken cancellationToken- )+ ) where T : UsenetResponse{ExceptionDispatchInfo? lastException = null;- var lastSuccessfulProviderContext = cancellationToken.GetContext<LastSuccessfulProviderContext>();- var lastSuccessfulProvider = lastSuccessfulProviderContext?.Provider;- var orderedProviders = GetOrderedProviders(lastSuccessfulProvider);- T? result = default;- foreach (var provider in orderedProviders)+ var orderedProviders = GetOrderedProviders();+ for (var i = 0; i < orderedProviders.Count; i++){cancellationToken.ThrowIfCancellationRequested();+ var provider = orderedProviders[i];+ var isLastProvider = i == orderedProviders.Count - 1;- if (lastException is not null && lastException.SourceException is not UsenetArticleNotFoundException)+ if (lastException is not null){var msg = lastException.SourceException.Message;Log.Debug($"Encountered error during NNTP Operation: `{msg}`. Trying another provider.");@@ -89,12 +140,12 @@ public class MultiProviderNntpClient(List<MultiConnectionNntpClient> providers)try{- result = await task.Invoke(provider).ConfigureAwait(false);- if (result is NntpStatResponse r && r.ResponseType != NntpStatResponseType.ArticleExists)- throw new UsenetArticleNotFoundException(r.MessageId.Value);+ var result = await task.Invoke(provider).ConfigureAwait(false);++ // if no article with that message-id is found, try again with the next provider.+ if (!isLastProvider && result.ResponseType == UsenetResponseType.NoArticleWithThatMessageId)+ continue;- if (lastSuccessfulProviderContext is not null && lastSuccessfulProvider != provider)- lastSuccessfulProviderContext.Provider = provider;return result;}catch (Exception e) when (e is not OperationCanceledException and not TaskCanceledException)@@ -103,24 +154,17 @@ public class MultiProviderNntpClient(List<MultiConnectionNntpClient> providers)}}- if (result is NntpStatResponse)- return result;-lastException?.Throw();throw new Exception("There are no usenet providers configured.");}- private IEnumerable<MultiConnectionNntpClient> GetOrderedProviders(MultiConnectionNntpClient? preferredProvider)+ private List<MultiConnectionNntpClient> GetOrderedProviders(){return providers.Where(x => x.ProviderType != ProviderType.Disabled).OrderBy(x => x.ProviderType)- .ThenByDescending(x => x.IdleConnections)- .ThenByDescending(x => x.RemainingSemaphoreSlots)- .Prepend(preferredProvider)- .Where(x => x is not null)- .Select(x => x!)- .Distinct();+ .ThenByDescending(x => x.AvailableConnections)+ .ToList();}public void Dispose()diff --git a/backend/Clients/Usenet/ThreadSafeNntpClient.cs b/backend/Clients/Usenet/ThreadSafeNntpClient.csdeleted file mode 100644index 92a3d050..00000000--- a/backend/Clients/Usenet/ThreadSafeNntpClient.cs+++ /dev/null@@ -1,155 +0,0 @@-using NzbWebDAV.Clients.Usenet.Models;-using NzbWebDAV.Exceptions;-using NzbWebDAV.Extensions;-using NzbWebDAV.Streams;-using Usenet.Nntp;-using Usenet.Nntp.Models;-using Usenet.Nntp.Responses;-using Usenet.Nzb;-using Usenet.Yenc;--namespace NzbWebDAV.Clients.Usenet;--public class ThreadSafeNntpClient : INntpClient-{- private readonly NntpConnection _connection;- private readonly NntpClient _client;- private readonly SemaphoreSlim _semaphore = new(1, 1);-- public ThreadSafeNntpClient()- {- _connection = new NntpConnection();- _client = new NntpClient(_connection);- }-- public Task<bool> ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken)- {- return Synchronized(() => _client.ConnectAsync(host, port, useSsl), cancellationToken);- }-- public Task<bool> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken)- {- return Synchronized(() => _client.Authenticate(user, pass), cancellationToken);- }-- public Task<NntpStatResponse> StatAsync(string segmentId, CancellationToken cancellationToken)- {- return Synchronized(() => _client.Stat(new NntpMessageId(segmentId)), cancellationToken);- }-- public Task<NntpDateResponse> DateAsync(CancellationToken cancellationToken)- {- return Synchronized(() => _client.Date(), cancellationToken);- }-- public Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken)- {- return Synchronized(() =>- {- var headResponse = _client.Head(new NntpMessageId(segmentId));- if (headResponse?.Article?.Headers == null) throw new UsenetArticleNotFoundException(segmentId);- return new UsenetArticleHeaders(headResponse.Article.Headers);- }, cancellationToken);- }-- public async Task<YencHeaderStream> GetSegmentStreamAsync- (- string segmentId,- bool includeHeaders,- CancellationToken cancellationToken- )- {- await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);- return await Task.Run(() =>- {- try- {- cancellationToken.ThrowIfCancellationRequested();- var article = GetArticle(segmentId, includeHeaders);- var stream = YencStreamDecoder.Decode(article.Body);- return new YencHeaderStream(- stream.Header,- article.Headers,- new BufferToEndStream(stream.OnDispose(OnDispose))- );-- // we only want to release the semaphore once the stream is disposed.- void OnDispose() => _semaphore.Release();- }- catch (Exception)- {- // or if there is an error getting the stream itself.- _semaphore.Release();- throw;- }- }).ConfigureAwait(false);- }-- public async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)- {- await using var stream = await GetSegmentStreamAsync(new NntpMessageId(segmentId), false, cancellationToken)- .ConfigureAwait(false);- return stream.Header;- }-- public async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)- {- if (file.Segments.Count == 0) return 0;- var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId.Value, cancellationToken)- .ConfigureAwait(false);- return header.PartOffset + header.PartSize;- }-- public async Task WaitForReady(CancellationToken cancellationToken)- {- await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);- _semaphore.Release();- }-- private Task<T> Synchronized<T>(Func<T> run, CancellationToken cancellationToken)- {- return Synchronized(() => Task.Run(run, cancellationToken), cancellationToken);- }-- private async Task<T> Synchronized<T>(Func<Task<T>> run, CancellationToken cancellationToken)- {- await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);- try- {- return await run().ConfigureAwait(false);- }- finally- {- _semaphore.Release();- }- }-- private UsenetArticle GetArticle(string segmentId, bool includeHeaders)- {- if (includeHeaders)- {- var articleResponse = _client.Article(new NntpMessageId(segmentId));- if (articleResponse?.Article?.Body == null) throw new UsenetArticleNotFoundException(segmentId);- return new UsenetArticle()- {- Headers = new UsenetArticleHeaders(articleResponse.Article.Headers),- Body = articleResponse.Article.Body- };- }-- var bodyResponse = _client.Body(new NntpMessageId(segmentId));- if (bodyResponse?.Article?.Body == null) throw new UsenetArticleNotFoundException(segmentId);- return new UsenetArticle()- {- Headers = null,- Body = bodyResponse.Article.Body- };- }-- public void Dispose()- {- _semaphore.Dispose();- _connection.Dispose();- GC.SuppressFinalize(this);- }-}\ No newline at end of filediff --git a/backend/Clients/Usenet/UsenetStreamingClient.cs b/backend/Clients/Usenet/UsenetStreamingClient.csindex ea5cb325..3625d3fa 100644--- a/backend/Clients/Usenet/UsenetStreamingClient.cs+++ b/backend/Clients/Usenet/UsenetStreamingClient.cs@@ -1,36 +1,16 @@using System.Text.Json;-using Microsoft.Extensions.Caching.Memory;using NzbWebDAV.Clients.Usenet.Connections;-using NzbWebDAV.Clients.Usenet.Models;using NzbWebDAV.Config;-using NzbWebDAV.Exceptions;-using NzbWebDAV.Extensions;-using NzbWebDAV.Streams;using NzbWebDAV.Websocket;-using Usenet.Nntp.Responses;-using Usenet.Nzb;namespace NzbWebDAV.Clients.Usenet;-public class UsenetStreamingClient+public class UsenetStreamingClient : WrappingNntpClient{- private readonly CachingNntpClient _client;- private readonly WebsocketManager _websocketManager;-public UsenetStreamingClient(ConfigManager configManager, WebsocketManager websocketManager)+ : base(CreateMultiProviderClient(configManager.GetUsenetProviderConfig(), websocketManager)){- // initialize private members- _websocketManager = websocketManager;-- // get connection settings from config-manager- var providerConfig = configManager.GetUsenetProviderConfig();-- // initialize the nntp-client- var multiProviderClient = CreateMultiProviderClient(providerConfig);- var cache = new MemoryCache(new MemoryCacheOptions() { SizeLimit = 8192 });- _client = new CachingNntpClient(multiProviderClient, cache);-- // when config changes, update the connection-pool+ // when config changes, create a new MultiProviderClient to use instead.configManager.OnConfigChanged += (_, configEventArgs) =>{// if unrelated config changed, do nothing@@ -38,135 +18,69 @@ public class UsenetStreamingClient// update the connection-pool according to the new configvar newProviderConfig = JsonSerializer.Deserialize<UsenetProviderConfig>(rawConfig);- var newMultiProviderClient = CreateMultiProviderClient(newProviderConfig!);- _client.UpdateUnderlyingClient(newMultiProviderClient);+ var newMultiProviderClient = CreateMultiProviderClient(newProviderConfig!, websocketManager);+ ReplaceUnderlyingClient(newMultiProviderClient);};}- public async Task CheckAllSegmentsAsync- (- IEnumerable<string> segmentIds,- int concurrency,- IProgress<int>? progress = null,- CancellationToken cancellationToken = default- )- {- using var childCt = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);- using var _1 = childCt.Token.SetScopedContext(cancellationToken.GetContext<ReservedPooledConnectionsContext>());- using var _2 = childCt.Token.SetScopedContext(cancellationToken.GetContext<LastSuccessfulProviderContext>());- var token = childCt.Token;-- var tasks = segmentIds- .Select(async x => (- SegmentId: x,- Result: await _client.StatAsync(x, token).ConfigureAwait(false)- ))- .WithConcurrencyAsync(concurrency);-- var processed = 0;- await foreach (var task in tasks.ConfigureAwait(false))- {- progress?.Report(++processed);- if (task.Result.ResponseType == NntpStatResponseType.ArticleExists) continue;- await childCt.CancelAsync().ConfigureAwait(false);- throw new UsenetArticleNotFoundException(task.SegmentId);- }- }-- public async Task<NzbFileStream> GetFileStream(NzbFile nzbFile, int concurrentConnections, CancellationToken ct)- {- var segmentIds = nzbFile.GetSegmentIds();- var fileSize = await _client.GetFileSizeAsync(nzbFile, ct).ConfigureAwait(false);- return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);- }-- public NzbFileStream GetFileStream(NzbFile nzbFile, long fileSize, int concurrentConnections)- {- return new NzbFileStream(nzbFile.GetSegmentIds(), fileSize, _client, concurrentConnections);- }-- public NzbFileStream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections)- {- return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);- }-- public Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, bool includeHeaders, CancellationToken ct)- {- return _client.GetSegmentStreamAsync(segmentId, includeHeaders, ct);- }-- public Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)- {- return _client.GetFileSizeAsync(file, cancellationToken);- }-- public Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken)- {- return _client.GetArticleHeadersAsync(segmentId, cancellationToken);- }-- private ConnectionPool<INntpClient> CreateNewConnectionPool+ private static MultiProviderNntpClient CreateMultiProviderClient(- int maxConnections,- ExtendedSemaphoreSlim pooledSemaphore,- Func<CancellationToken, ValueTask<INntpClient>> connectionFactory,- EventHandler<ConnectionPoolStats.ConnectionPoolChangedEventArgs> onConnectionPoolChanged+ UsenetProviderConfig providerConfig,+ WebsocketManager websocketManager){- var connectionPool = new ConnectionPool<INntpClient>(maxConnections, pooledSemaphore, connectionFactory);- connectionPool.OnConnectionPoolChanged += onConnectionPoolChanged;- var args = new ConnectionPoolStats.ConnectionPoolChangedEventArgs(0, 0, maxConnections);- onConnectionPoolChanged(connectionPool, args);- return connectionPool;- }-- private MultiProviderNntpClient CreateMultiProviderClient(UsenetProviderConfig providerConfig)- {- var connectionPoolStats = new ConnectionPoolStats(providerConfig, _websocketManager);- var totalPooledConnectionCount = providerConfig.TotalPooledConnections;- var pooledSemaphore = new ExtendedSemaphoreSlim(totalPooledConnectionCount, totalPooledConnectionCount);+ var connectionPoolStats = new ConnectionPoolStats(providerConfig, websocketManager);var providerClients = providerConfig.Providers.Select((provider, index) => CreateProviderClient(provider,- connectionPoolStats.GetOnConnectionPoolChanged(index),- pooledSemaphore+ connectionPoolStats.GetOnConnectionPoolChanged(index))).ToList();return new MultiProviderNntpClient(providerClients);}- private MultiConnectionNntpClient CreateProviderClient+ private static MultiConnectionNntpClient CreateProviderClient(UsenetProviderConfig.ConnectionDetails connectionDetails,- EventHandler<ConnectionPoolStats.ConnectionPoolChangedEventArgs> onConnectionPoolChanged,- ExtendedSemaphoreSlim pooledSemaphore+ EventHandler<ConnectionPoolStats.ConnectionPoolChangedEventArgs> onConnectionPoolChanged){var connectionPool = CreateNewConnectionPool(maxConnections: connectionDetails.MaxConnections,- pooledSemaphore: pooledSemaphore,connectionFactory: ct => CreateNewConnection(connectionDetails, ct),onConnectionPoolChanged);return new MultiConnectionNntpClient(connectionPool, connectionDetails.Type);}+ private static ConnectionPool<INntpClient> CreateNewConnectionPool+ (+ int maxConnections,+ Func<CancellationToken, ValueTask<INntpClient>> connectionFactory,+ EventHandler<ConnectionPoolStats.ConnectionPoolChangedEventArgs> onConnectionPoolChanged+ )+ {+ var connectionPool = new ConnectionPool<INntpClient>(maxConnections, connectionFactory);+ connectionPool.OnConnectionPoolChanged += onConnectionPoolChanged;+ var args = new ConnectionPoolStats.ConnectionPoolChangedEventArgs(0, 0, maxConnections);+ onConnectionPoolChanged(connectionPool, args);+ return connectionPool;+ }+public static async ValueTask<INntpClient> CreateNewConnection(UsenetProviderConfig.ConnectionDetails connectionDetails,- CancellationToken cancellationToken+ CancellationToken ct){- var connection = new ThreadSafeNntpClient();+ var connection = new BaseNntpClient();var host = connectionDetails.Host;var port = connectionDetails.Port;var useSsl = connectionDetails.UseSsl;var user = connectionDetails.User;var pass = connectionDetails.Pass;- if (!await connection.ConnectAsync(host, port, useSsl, cancellationToken).ConfigureAwait(false))- throw new CouldNotConnectToUsenetException("Could not connect to usenet host. Check connection settings.");- if (!await connection.AuthenticateAsync(user, pass, cancellationToken).ConfigureAwait(false))- throw new CouldNotLoginToUsenetException("Could not login to usenet host. Check username and password.");+ await connection.ConnectAsync(host, port, useSsl, ct).ConfigureAwait(false);+ await connection.AuthenticateAsync(user, pass, ct).ConfigureAwait(false);return connection;}}\ No newline at end of filediff --git a/backend/Clients/Usenet/WrappingNntpClient.cs b/backend/Clients/Usenet/WrappingNntpClient.csindex 7ca4145b..d04daea9 100644--- a/backend/Clients/Usenet/WrappingNntpClient.cs+++ b/backend/Clients/Usenet/WrappingNntpClient.cs@@ -1,75 +1,76 @@using NzbWebDAV.Clients.Usenet.Models;-using NzbWebDAV.Streams;-using Usenet.Nntp.Responses;-using Usenet.Nzb;-using Usenet.Yenc;+using UsenetSharp.Models;namespace NzbWebDAV.Clients.Usenet;-public abstract class WrappingNntpClient(INntpClient client) : INntpClient+public class WrappingNntpClient(INntpClient client) : INntpClient{- protected INntpClient Client = client;+ private INntpClient _client = client;- public virtual Task<bool> ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken)+ public Task ConnectAsync(string host, int port, bool useSsl, CancellationToken cancellationToken){- return Client.ConnectAsync(host, port, useSsl, cancellationToken);+ return _client.ConnectAsync(host, port, useSsl, cancellationToken);}- public virtual Task<bool> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken)+ public Task<UsenetResponse> AuthenticateAsync(string user, string pass, CancellationToken cancellationToken){- return Client.AuthenticateAsync(user, pass, cancellationToken);+ return _client.AuthenticateAsync(user, pass, cancellationToken);}- public virtual Task<NntpStatResponse> StatAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetStatResponse> StatAsync(SegmentId segmentId, CancellationToken cancellationToken){- return Client.StatAsync(segmentId, cancellationToken);+ return _client.StatAsync(segmentId, cancellationToken);}- public virtual Task<NntpDateResponse> DateAsync(CancellationToken cancellationToken)+ public Task<UsenetHeadResponse> HeadAsync(SegmentId segmentId, CancellationToken cancellationToken){- return Client.DateAsync(cancellationToken);+ return _client.HeadAsync(segmentId, cancellationToken);}- public Task<UsenetArticleHeaders> GetArticleHeadersAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId, CancellationToken cancellationToken){- return Client.GetArticleHeadersAsync(segmentId, cancellationToken);+ return _client.DecodedBodyAsync(segmentId, cancellationToken);}- public virtual Task<YencHeaderStream> GetSegmentStreamAsync- (- string segmentId,- bool includeHeaders,- CancellationToken cancellationToken- )+ public Task<UsenetDecodedArticleResponse> DecodedArticleAsync(SegmentId segmentId,+ CancellationToken cancellationToken){- return Client.GetSegmentStreamAsync(segmentId, includeHeaders, cancellationToken);+ return _client.DecodedArticleAsync(segmentId, cancellationToken);}- public virtual Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)+ public Task<UsenetDateResponse> DateAsync(CancellationToken cancellationToken){- return Client.GetSegmentYencHeaderAsync(segmentId, cancellationToken);+ return _client.DateAsync(cancellationToken);}- public virtual Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)+ public Task WaitForReadyAsync(CancellationToken cancellationToken){- return Client.GetFileSizeAsync(file, cancellationToken);+ return _client.WaitForReadyAsync(cancellationToken);}- public virtual Task WaitForReady(CancellationToken cancellationToken)+ public Task<UsenetDecodedBodyResponse> DecodedBodyAsync(SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain, CancellationToken cancellationToken){- return Client.WaitForReady(cancellationToken);+ return _client.DecodedBodyAsync(segmentId, onConnectionReadyAgain, cancellationToken);}- public void UpdateUnderlyingClient(INntpClient client)+ public Task<UsenetDecodedArticleResponse> DecodedArticleAsync(SegmentId segmentId,+ Action<ArticleBodyResult>? onConnectionReadyAgain, CancellationToken cancellationToken){- var oldClient = Client;- Client = client;- oldClient.Dispose();+ return _client.DecodedArticleAsync(segmentId, onConnectionReadyAgain, cancellationToken);+ }++ protected void ReplaceUnderlyingClient(INntpClient client)+ {+ var old = _client;+ _client = client;+ if (old is IDisposable disposable)+ disposable.Dispose();}public void Dispose(){- Client.Dispose();+ _client.Dispose();GC.SuppressFinalize(this);}}\ No newline at end of filediff --git a/backend/Exceptions/CouldNotConnectToUsenetException.cs b/backend/Exceptions/CouldNotConnectToUsenetException.csindex a5ce507b..f3b22161 100644--- a/backend/Exceptions/CouldNotConnectToUsenetException.cs+++ b/backend/Exceptions/CouldNotConnectToUsenetException.cs@@ -1,5 +1,6 @@namespace NzbWebDAV.Exceptions;-public class CouldNotConnectToUsenetException(string message) : RetryableDownloadException(message)+public class CouldNotConnectToUsenetException(string message, Exception? innerException = null)+ : RetryableDownloadException(message, innerException){}\ No newline at end of filediff --git a/backend/Exceptions/CouldNotLoginToUsenetException.cs b/backend/Exceptions/CouldNotLoginToUsenetException.csindex 30075bd2..69ca5d4a 100644--- a/backend/Exceptions/CouldNotLoginToUsenetException.cs+++ b/backend/Exceptions/CouldNotLoginToUsenetException.cs@@ -1,5 +1,6 @@namespace NzbWebDAV.Exceptions;-public class CouldNotLoginToUsenetException(string message) : RetryableDownloadException(message)+public class CouldNotLoginToUsenetException(string message, Exception? innerException = null)+ : RetryableDownloadException(message, innerException){}\ No newline at end of filediff --git a/backend/Exceptions/RetryableDownloadException.cs b/backend/Exceptions/RetryableDownloadException.csindex 1a9d768d..643b33ed 100644--- a/backend/Exceptions/RetryableDownloadException.cs+++ b/backend/Exceptions/RetryableDownloadException.cs@@ -1,5 +1,6 @@namespace NzbWebDAV.Exceptions;-public class RetryableDownloadException(string message) : Exception(message)+public class RetryableDownloadException(string message, Exception? innerException = null)+ : Exception(message, innerException){}\ No newline at end of filediff --git a/backend/NzbWebDAV.csproj b/backend/NzbWebDAV.csprojindex 59f0bb7c..8bc94b7c 100644--- a/backend/NzbWebDAV.csproj+++ b/backend/NzbWebDAV.csproj@@ -19,6 +19,7 @@<PackageReference Include="Serilog.Sinks.Console" Version="6.0.1-dev-00953" /><PackageReference Include="SharpCompress" Version="0.39.0" /><PackageReference Include="Usenet" Version="3.1.0" />+ <PackageReference Include="UsenetSharp" Version="1.0.2" /></ItemGroup><ItemGroup>diff --git a/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.csindex bd8a96eb..eaac5aeb 100644--- a/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs+++ b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs@@ -5,7 +5,7 @@ using NzbWebDAV.Config;using NzbWebDAV.Exceptions;using NzbWebDAV.Extensions;using Usenet.Nzb;-using Usenet.Yenc;+using UsenetSharp.Models;namespace NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;@@ -14,9 +14,9 @@ public static class FetchFirstSegmentsSteppublic static async Task<List<NzbFileWithFirstSegment>> FetchFirstSegments(List<NzbFile> nzbFiles,- UsenetStreamingClient client,+ INntpClient client,ConfigManager configManager,- CancellationToken cancellationToken = default,+ CancellationToken cancellationToken,IProgress<int>? progress = null){@@ -30,22 +30,23 @@ public static class FetchFirstSegmentsStepprivate static async Task<NzbFileWithFirstSegment> FetchFirstSegment(NzbFile nzbFile,- UsenetStreamingClient client,- CancellationToken cancellationToken = default+ INntpClient client,+ CancellationToken cancellationToken){try{// get the first article streamvar firstSegment = nzbFile.Segments[0].MessageId.Value;- await using var stream = await client.GetSegmentStreamAsync(firstSegment, true, cancellationToken).ConfigureAwait(false);+ var article = await client.DecodedArticleAsync(firstSegment, cancellationToken).ConfigureAwait(false);+ await using var bodyStream = article.Stream!;// read up to the first 16KB from the streamvar totalRead = 0;var buffer = new byte[16 * 1024];while (totalRead < buffer.Length){- var read = await stream.ReadAsync(buffer.AsMemory(totalRead, buffer.Length - totalRead),+ var read = await bodyStream.ReadAsync(buffer.AsMemory(totalRead, buffer.Length - totalRead),cancellationToken).ConfigureAwait(false);if (read == 0) break;totalRead += read;@@ -56,14 +57,19 @@ public static class FetchFirstSegmentsStep? buffer.AsSpan(0, totalRead).ToArray(): buffer;+ // get the yencHeaders+ var yencHeaders = await bodyStream+ .GetYencHeadersAsync(cancellationToken)+ .ConfigureAwait(false);+// returnreturn new NzbFileWithFirstSegment{NzbFile = nzbFile,First16KB = first16KB,- Header = stream.Header,+ Header = yencHeaders,MissingFirstSegment = false,- ReleaseDate = stream.ArticleHeaders!.Date+ ReleaseDate = article.ArticleHeaders!.Date};}catch (UsenetArticleNotFoundException)@@ -85,7 +91,7 @@ public static class FetchFirstSegmentsStepprivate static readonly byte[] Rar5Magic = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00];public required NzbFile NzbFile { get; init; }- public required YencHeader? Header { get; init; }+ public required UsenetYencHeader? Header { get; init; }public required byte[]? First16KB { get; init; }public required bool MissingFirstSegment { get; init; }public required DateTimeOffset ReleaseDate { get; init; }diff --git a/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.csindex d62213bd..50302bea 100644--- a/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs+++ b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs@@ -11,7 +11,7 @@ public static class GetPar2FileDescriptorsSteppublic static async Task<List<FileDesc>> GetPar2FileDescriptors(List<FetchFirstSegmentsStep.NzbFileWithFirstSegment> files,- UsenetStreamingClient client,+ INntpClient client,CancellationToken cancellationToken = default){diff --git a/backend/Queue/FileProcessors/FileProcessor.cs b/backend/Queue/FileProcessors/FileProcessor.csindex d6c09b87..5c3a9aeb 100644--- a/backend/Queue/FileProcessors/FileProcessor.cs+++ b/backend/Queue/FileProcessors/FileProcessor.cs@@ -9,7 +9,7 @@ namespace NzbWebDAV.Queue.FileProcessors;public class FileProcessor(GetFileInfosStep.FileInfo fileInfo,- UsenetStreamingClient usenet,+ INntpClient usenet,CancellationToken ct) : BaseProcessor{diff --git a/backend/Queue/FileProcessors/MultipartMkvProcessor.cs b/backend/Queue/FileProcessors/MultipartMkvProcessor.csindex 61eab86b..ad4f4710 100644--- a/backend/Queue/FileProcessors/MultipartMkvProcessor.cs+++ b/backend/Queue/FileProcessors/MultipartMkvProcessor.cs@@ -10,13 +10,13 @@ namespace NzbWebDAV.Queue.FileProcessors;public class MultipartMkvProcessor : BaseProcessor{private readonly List<GetFileInfosStep.FileInfo> _fileInfos;- private readonly UsenetStreamingClient _client;+ private readonly INntpClient _client;private readonly CancellationToken _ct;public MultipartMkvProcessor(List<GetFileInfosStep.FileInfo> fileInfos,- UsenetStreamingClient client,+ INntpClient client,CancellationToken ct){diff --git a/backend/Queue/FileProcessors/RarProcessor.cs b/backend/Queue/FileProcessors/RarProcessor.csindex f4fe0f42..4a86b9a1 100644--- a/backend/Queue/FileProcessors/RarProcessor.cs+++ b/backend/Queue/FileProcessors/RarProcessor.cs@@ -12,7 +12,7 @@ namespace NzbWebDAV.Queue.FileProcessors;public class RarProcessor(GetFileInfosStep.FileInfo fileInfo,- UsenetStreamingClient usenet,+ INntpClient usenet,string? password,CancellationToken ct) : BaseProcessordiff --git a/backend/Queue/FileProcessors/SevenZipProcessor.cs b/backend/Queue/FileProcessors/SevenZipProcessor.csindex e23f300e..88ddb31b 100644--- a/backend/Queue/FileProcessors/SevenZipProcessor.cs+++ b/backend/Queue/FileProcessors/SevenZipProcessor.cs@@ -14,14 +14,14 @@ namespace NzbWebDAV.Queue.FileProcessors;public class SevenZipProcessor : BaseProcessor{private readonly List<GetFileInfosStep.FileInfo> _fileInfos;- private readonly UsenetStreamingClient _client;+ private readonly INntpClient _client;private readonly string? _archivePassword;private readonly CancellationToken _ct;public SevenZipProcessor(List<GetFileInfosStep.FileInfo> fileInfos,- UsenetStreamingClient client,+ INntpClient client,string? archivePassword,CancellationToken ct)diff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.csindex f96b63ec..6b4987a5 100644--- a/backend/Queue/QueueItemProcessor.cs+++ b/backend/Queue/QueueItemProcessor.cs@@ -3,7 +3,6 @@ using Microsoft.EntityFrameworkCore;using NzbWebDAV.Api.SabControllers.GetHistory;using NzbWebDAV.Clients.RadarrSonarr;using NzbWebDAV.Clients.Usenet;-using NzbWebDAV.Clients.Usenet.Connections;using NzbWebDAV.Config;using NzbWebDAV.Database;using NzbWebDAV.Database.Models;@@ -26,7 +25,7 @@ public class QueueItemProcessor(QueueItem queueItem,QueueNzbContents queueNzbContents,DavDatabaseClient dbClient,- UsenetStreamingClient usenetClient,+ INntpClient usenetClient,ConfigManager configManager,WebsocketManager websocketManager,HealthCheckService healthCheckService,@@ -108,10 +107,7 @@ public class QueueItemProcessor(}// ensure we don't use more than max-queue-connections- var providerConfig = configManager.GetUsenetProviderConfig();var concurrency = configManager.GetMaxQueueConnections();- var reservedConnections = providerConfig.TotalPooledConnections - concurrency;- using var _ = ct.SetScopedContext(new ReservedPooledConnectionsContext(reservedConnections));// read the nzb documentvar documentBytes = Encoding.UTF8.GetBytes(queueNzbContents.NzbContents);diff --git a/backend/Services/HealthCheckService.cs b/backend/Services/HealthCheckService.csindex dd142475..77e9cad9 100644--- a/backend/Services/HealthCheckService.cs+++ b/backend/Services/HealthCheckService.cs@@ -1,6 +1,5 @@using Microsoft.EntityFrameworkCore;using NzbWebDAV.Clients.Usenet;-using NzbWebDAV.Clients.Usenet.Connections;using NzbWebDAV.Config;using NzbWebDAV.Database;using NzbWebDAV.Database.Models;@@ -18,7 +17,7 @@ namespace NzbWebDAV.Services;public class HealthCheckService{private readonly ConfigManager _configManager;- private readonly UsenetStreamingClient _usenetClient;+ private readonly INntpClient _usenetClient;private readonly WebsocketManager _websocketManager;private readonly CancellationToken _cancellationToken = SigtermUtil.GetCancellationToken();@@ -60,12 +59,7 @@ public class HealthCheckService// get concurrencyvar concurrency = _configManager.GetMaxRepairConnections();-- // set reserved-connections contextusing var cts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken);- var providerConfig = _configManager.GetUsenetProviderConfig();- var reservedConnections = providerConfig.TotalPooledConnections - concurrency;- using var _ = cts.Token.SetScopedContext(new ReservedPooledConnectionsContext(reservedConnections));// get the davItem to health-checkawait using var dbContext = new DavDatabaseContext();@@ -174,7 +168,8 @@ public class HealthCheckService{var firstSegmentId = StringUtil.EmptyToNull(segments.FirstOrDefault());if (firstSegmentId == null) return;- var articleHeaders = await _usenetClient.GetArticleHeadersAsync(firstSegmentId, ct).ConfigureAwait(false);+ var articleHeadersResponse = await _usenetClient.HeadAsync(firstSegmentId, ct).ConfigureAwait(false);+ var articleHeaders = articleHeadersResponse.ArticleHeaders!;davItem.ReleaseDate = articleHeaders.Date;}diff --git a/backend/Streams/DavMultipartFileStream.cs b/backend/Streams/DavMultipartFileStream.csindex 87e677f1..87ae3235 100644--- a/backend/Streams/DavMultipartFileStream.cs+++ b/backend/Streams/DavMultipartFileStream.cs@@ -7,7 +7,7 @@ namespace NzbWebDAV.Streams;public class DavMultipartFileStream(DavMultipartFile.FilePart[] fileParts,- UsenetStreamingClient usenet,+ INntpClient usenet,int concurrentConnections) : Stream{diff --git a/backend/Streams/MultipartFileStream.cs b/backend/Streams/MultipartFileStream.csindex e8cbb62c..a22d5205 100644--- a/backend/Streams/MultipartFileStream.cs+++ b/backend/Streams/MultipartFileStream.cs@@ -7,7 +7,7 @@ namespace NzbWebDAV.Streams;public class MultipartFileStream : Stream{private bool _isDisposed;- private readonly UsenetStreamingClient _client;+ private readonly INntpClient _client;private readonly MultipartFile _multipartFile;private Stream? _currentStream;private long _position = 0;@@ -23,7 +23,7 @@ public class MultipartFileStream : Streamset => throw new NotSupportedException();}- public MultipartFileStream(MultipartFile multipartFile, UsenetStreamingClient client)+ public MultipartFileStream(MultipartFile multipartFile, INntpClient client){_multipartFile = multipartFile;_client = client;diff --git a/backend/Streams/NzbFileStream.cs b/backend/Streams/NzbFileStream.csindex 75c89323..97f26bbc 100644--- a/backend/Streams/NzbFileStream.cs+++ b/backend/Streams/NzbFileStream.cs@@ -28,7 +28,8 @@ public class NzbFileStream(public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken){- if (_innerStream == null) _innerStream = await GetFileStream(_position, cancellationToken).ConfigureAwait(false);+ if (_innerStream == null)+ _innerStream = await GetFileStream(_position, cancellationToken).ConfigureAwait(false);var read = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);_position += read;return read;@@ -76,7 +77,7 @@ public class NzbFileStream(new LongRange(0, fileSize),async (guess) =>{- var header = await client.GetSegmentYencHeaderAsync(fileSegmentIds[guess], ct).ConfigureAwait(false);+ var header = await client.GetYencHeadersAsync(fileSegmentIds[guess], ct).ConfigureAwait(false);return new LongRange(header.PartOffset, header.PartOffset + header.PartSize);},ct@@ -96,7 +97,7 @@ public class NzbFileStream({return new CombinedStream(fileSegmentIds[firstSegmentIndex..]- .Select(async x => (Stream)await client.GetSegmentStreamAsync(x, false, ct).ConfigureAwait(false))+ .Select(async x => (Stream)(await client.DecodedBodyAsync(x, ct).ConfigureAwait(false)).Stream).WithConcurrency(concurrentConnections));}diff --git a/backend/Streams/YencHeaderStream.cs b/backend/Streams/YencHeaderStream.csdeleted file mode 100644index d4a23951..00000000--- a/backend/Streams/YencHeaderStream.cs+++ /dev/null@@ -1,50 +0,0 @@-using NzbWebDAV.Clients.Usenet.Models;-using Usenet.Yenc;--namespace NzbWebDAV.Streams;--public class YencHeaderStream(YencHeader header, UsenetArticleHeaders? articleHeaders, Stream stream) : Stream-{- public YencHeader Header => header;- public UsenetArticleHeaders? ArticleHeaders => articleHeaders;- private bool _disposed;-- public override void Flush() => stream.Flush();- public override int Read(byte[] buffer, int offset, int count) => stream.Read(buffer, offset, count);-- public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) =>- stream.ReadAsync(buffer, offset, count, cancellationToken);-- public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) =>- stream.ReadAsync(buffer, cancellationToken);-- public override long Seek(long offset, SeekOrigin origin) => stream.Seek(offset, origin);- public override void SetLength(long value) => stream.SetLength(value);- public override void Write(byte[] buffer, int offset, int count) => stream.Write(buffer, offset, count);-- public override bool CanRead => stream.CanRead;- public override bool CanSeek => stream.CanSeek;- public override bool CanWrite => stream.CanWrite;- public override long Length => Header.PartSize;-- public override long Position- {- get => stream.Position;- set => stream.Position = value;- }-- protected override void Dispose(bool disposing)- {- if (_disposed) return;- stream.Dispose();- _disposed = true;- }-- public override async ValueTask DisposeAsync()- {- if (_disposed) return;- await stream.DisposeAsync().ConfigureAwait(false);- _disposed = true;- GC.SuppressFinalize(this);- }-}\ No newline at end of filediff --git a/backend/WebDav/Base/BaseStoreStreamFile.cs b/backend/WebDav/Base/BaseStoreStreamFile.csindex 2d946f39..0d761f9b 100644--- a/backend/WebDav/Base/BaseStoreStreamFile.cs+++ b/backend/WebDav/Base/BaseStoreStreamFile.cs@@ -1,21 +1,11 @@-using NzbWebDAV.Clients.Usenet.Connections;-using NzbWebDAV.Extensions;-using NzbWebDAV.Streams;--namespace NzbWebDAV.WebDav.Base;+namespace NzbWebDAV.WebDav.Base;public abstract class BaseStoreStreamFile() : BaseStoreReadonlyItem{- public abstract Task<Stream> GetStreamAsync(CancellationToken cancellationToken);+ protected abstract Task<Stream> GetStreamAsync(CancellationToken cancellationToken);- public override async Task<Stream> GetReadableStreamAsync(CancellationToken cancellationToken)+ public override Task<Stream> GetReadableStreamAsync(CancellationToken cancellationToken){- var stream = await GetStreamAsync(cancellationToken).ConfigureAwait(false);- var scopedContext = cancellationToken.SetScopedContext(new LastSuccessfulProviderContext());- return new DisposableCallbackStream- (- stream,- () => scopedContext.Dispose()- );+ return GetStreamAsync(cancellationToken);}}\ No newline at end of filediff --git a/backend/WebDav/DatabaseStoreMultipartFile.cs b/backend/WebDav/DatabaseStoreMultipartFile.csindex 24b6d61d..75930dbe 100644--- a/backend/WebDav/DatabaseStoreMultipartFile.cs+++ b/backend/WebDav/DatabaseStoreMultipartFile.cs@@ -23,7 +23,7 @@ public class DatabaseStoreMultipartFile(public override long FileSize => davMultipartFile.FileSize!.Value;public override DateTime CreatedAt => davMultipartFile.CreatedAt;- public override async Task<Stream> GetStreamAsync(CancellationToken ct)+ protected override async Task<Stream> GetStreamAsync(CancellationToken ct){// store the DavItem being accessed in the http contexthttpContext.Items["DavItem"] = davMultipartFile;diff --git a/backend/WebDav/DatabaseStoreNzbFile.cs b/backend/WebDav/DatabaseStoreNzbFile.csindex f850c408..7e11efb6 100644--- a/backend/WebDav/DatabaseStoreNzbFile.cs+++ b/backend/WebDav/DatabaseStoreNzbFile.cs@@ -11,7 +11,7 @@ public class DatabaseStoreNzbFile(DavItem davNzbFile,HttpContext httpContext,DavDatabaseClient dbClient,- UsenetStreamingClient usenetClient,+ INntpClient usenetClient,ConfigManager configManager) : BaseStoreStreamFile{@@ -21,7 +21,7 @@ public class DatabaseStoreNzbFile(public override long FileSize => davNzbFile.FileSize!.Value;public override DateTime CreatedAt => davNzbFile.CreatedAt;- public override async Task<Stream> GetStreamAsync(CancellationToken cancellationToken)+ protected override async Task<Stream> GetStreamAsync(CancellationToken cancellationToken){// store the DavItem being accessed in the http contexthttpContext.Items["DavItem"] = davNzbFile;diff --git a/backend/WebDav/DatabaseStoreRarFile.cs b/backend/WebDav/DatabaseStoreRarFile.csindex dd0cdf77..1af2b5fe 100644--- a/backend/WebDav/DatabaseStoreRarFile.cs+++ b/backend/WebDav/DatabaseStoreRarFile.cs@@ -23,7 +23,7 @@ public class DatabaseStoreRarFile(public override long FileSize => davRarFile.FileSize!.Value;public override DateTime CreatedAt => davRarFile.CreatedAt;- public override async Task<Stream> GetStreamAsync(CancellationToken ct)+ protected override async Task<Stream> GetStreamAsync(CancellationToken ct){// store the DavItem being accessed in the http contexthttpContext.Items["DavItem"] = davRarFile;