Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Deobfuscate filenames using par2 files.
nzbdav-dev47d51be09f32
diff --git a/backend/Clients/CachingNntpClient.cs b/backend/Clients/CachingNntpClient.csindex 59cf7b9c..2f70897b 100644--- a/backend/Clients/CachingNntpClient.cs+++ b/backend/Clients/CachingNntpClient.cs@@ -1,4 +1,5 @@using Microsoft.Extensions.Caching.Memory;+using NzbWebDAV.Streams;using Usenet.Nzb;using Usenet.Yenc;@@ -8,21 +9,34 @@ public class CachingNntpClient(INntpClient client, MemoryCache cache) : Wrapping{private readonly INntpClient _client = client;- public override async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)+ private readonly MemoryCacheEntryOptions _cacheOptions = new()+ {+ Size = 1,+ SlidingExpiration = TimeSpan.FromHours(3)+ };++ public override async Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, CancellationToken ct)+ {+ var cacheKey = segmentId;+ var stream = await _client.GetSegmentStreamAsync(segmentId, ct);+ cache.Set(cacheKey, stream.Header, _cacheOptions);+ return stream;+ }++ public override async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken ct){var cacheKey = segmentId;return (await cache.GetOrCreateAsync(cacheKey, cacheEntry =>{- cacheEntry.Size = 1;- cacheEntry.SlidingExpiration = TimeSpan.FromHours(3);- return _client.GetSegmentYencHeaderAsync(segmentId, cancellationToken);+ cacheEntry.SetOptions(_cacheOptions);+ return _client.GetSegmentYencHeaderAsync(segmentId, ct);})!)!;}- public override async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)+ public override async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken ct){if (file.Segments.Count == 0) return 0;- var header = await GetSegmentYencHeaderAsync(file.Segments[0].MessageId, cancellationToken);- return header.FileSize;+ var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId, ct);+ return header.PartOffset + header.PartSize;}}\ No newline at end of filediff --git a/backend/Clients/ThreadSafeNntpClient.cs b/backend/Clients/ThreadSafeNntpClient.csindex e36a91c3..95f6c27b 100644--- a/backend/Clients/ThreadSafeNntpClient.cs+++ b/backend/Clients/ThreadSafeNntpClient.cs@@ -77,8 +77,8 @@ public class ThreadSafeNntpClient : INntpClientpublic async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken){if (file.Segments.Count == 0) return 0;- var header = await GetSegmentYencHeaderAsync(file.Segments[0].MessageId.Value, cancellationToken);- return header.FileSize;+ var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId.Value, cancellationToken);+ return header.PartOffset + header.PartSize;}public async Task WaitForReady(CancellationToken cancellationToken)@@ -108,7 +108,7 @@ public class ThreadSafeNntpClient : INntpClientprivate IEnumerable<string> GetArticleBody(string segmentId){return _client.Body(new NntpMessageId(segmentId))?.Article?.Body- ?? throw new UsenetArticleNotFoundException($"Article with message-id {segmentId} not found.");+ ?? throw new UsenetArticleNotFoundException($"Article with message-id {segmentId} not found.");}public void Dispose()diff --git a/backend/Clients/UsenetStreamingClient.cs b/backend/Clients/UsenetStreamingClient.csindex fcbedab2..eda26a06 100644--- a/backend/Clients/UsenetStreamingClient.cs+++ b/backend/Clients/UsenetStreamingClient.cs@@ -2,12 +2,10 @@using NzbWebDAV.Clients.Connections;using NzbWebDAV.Config;using NzbWebDAV.Exceptions;-using NzbWebDAV.Extensions;using NzbWebDAV.Streams;using NzbWebDAV.Websocket;using Usenet.Nntp.Responses;using Usenet.Nzb;-using Usenet.Yenc;namespace NzbWebDAV.Clients;@@ -86,19 +84,19 @@ public class UsenetStreamingClientpublic async Task<NzbFileStream> GetFileStream(NzbFile nzbFile, int concurrentConnections, CancellationToken ct){- var firstSegmentId = nzbFile.GetOrderedSegmentIds().First();- var firstSegmentStream = await _client.GetSegmentStreamAsync(firstSegmentId, ct);- return new NzbFileStream(nzbFile, firstSegmentStream, _client, concurrentConnections);+ var segmentIds = nzbFile.Segments.Select(x => x.MessageId.Value).ToArray();+ var fileSize = await _client.GetFileSizeAsync(nzbFile, cancellationToken: ct);+ return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);}- public Stream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections)+ public NzbFileStream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections){return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);}- public Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)+ public Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, CancellationToken cancellationToken){- return _client.GetSegmentYencHeaderAsync(segmentId, cancellationToken);+ return _client.GetSegmentStreamAsync(segmentId, cancellationToken);}public Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)diff --git a/backend/Extensions/IAsyncEnumerableTaskExtensions.cs b/backend/Extensions/IAsyncEnumerableTaskExtensions.csnew file mode 100644index 00000000..f4969e53--- /dev/null+++ b/backend/Extensions/IAsyncEnumerableTaskExtensions.cs@@ -0,0 +1,24 @@+// ReSharper disable InconsistentNaming++namespace NzbWebDAV.Extensions;++public static class IAsyncEnumerableTaskExtensions+{+ public static async Task<List<T>> GetAllAsync<T>+ (+ this IAsyncEnumerable<T> asyncEnumerable,+ CancellationToken ct = default,+ IProgress<int>? progress = null+ )+ {+ var done = 0;+ var results = new List<T>();+ await foreach (var result in asyncEnumerable.WithCancellation(ct))+ {+ results.Add(result);+ progress?.Report(++done);+ }++ return results;+ }+}\ No newline at end of filediff --git a/backend/Extensions/IEnumerableTaskExtensions.cs b/backend/Extensions/IEnumerableTaskExtensions.csindex 0831d711..cf9d3cd9 100644--- a/backend/Extensions/IEnumerableTaskExtensions.cs+++ b/backend/Extensions/IEnumerableTaskExtensions.cs@@ -61,4 +61,31 @@ public static class IEnumerableTaskExtensions}}}++ public static async IAsyncEnumerable<T> WithConcurrencyAsync<T>+ (+ this IEnumerable<Task<T>> tasks,+ int concurrency+ )+ {+ if (concurrency < 1)+ throw new ArgumentException("concurrency must be greater than zero.");++ var runningTasks = new HashSet<Task<T>>();+ foreach (var task in tasks)+ {+ runningTasks.Add(task);+ if (runningTasks.Count < concurrency) continue;+ var completedTask = await Task.WhenAny(runningTasks);+ runningTasks.Remove(completedTask);+ yield return await completedTask;+ }++ while (runningTasks.Count > 0)+ {+ var completedTask = await Task.WhenAny(runningTasks);+ runningTasks.Remove(completedTask);+ yield return await completedTask;+ }+ }}\ No newline at end of filediff --git a/backend/Extensions/NzbFileExtensions.cs b/backend/Extensions/NzbFileExtensions.csindex 10f11b36..1f01211b 100644--- a/backend/Extensions/NzbFileExtensions.cs+++ b/backend/Extensions/NzbFileExtensions.cs@@ -1,5 +1,4 @@using System.Text.RegularExpressions;-using NzbWebDAV.Clients;using Usenet.Nzb;namespace NzbWebDAV.Extensions;@@ -14,38 +13,6 @@ public static class NzbFileExtensions.ToArray();}- public static async Task<string> GetFileName- (- this NzbFile file,- UsenetStreamingClient client,- CancellationToken ct = default- )- {- // prioritize subject filename with fallback to header filename.- // unless header filename has a valid extension and subject filename doesn't.- var subjectFilename = file.GetSubjectFileName();- if (subjectFilename == "") return await file.GetHeaderFileName(client, ct);- var subjectFilenameExtension = Path.GetExtension(subjectFilename).TrimStart('.');- if (subjectFilenameExtension.Length is >= 2 and <= 4) return subjectFilename;- var headerFilename = await file.GetHeaderFileName(client, ct);- var headerFilenameExtension = Path.GetExtension(headerFilename).TrimStart('.');- if (headerFilenameExtension.Length is >=2 and <=4) return headerFilename;- return subjectFilename;- }-- public static async Task<string> GetHeaderFileName- (- this NzbFile file,- UsenetStreamingClient client,- CancellationToken ct = default- )- {- if (file.Segments.Count == 0) return "";- var firstSegment = file.Segments[0].MessageId.Value;- var header = await client.GetSegmentYencHeaderAsync(firstSegment, default);- return header.FileName;- }-public static string GetSubjectFileName(this NzbFile file){return GetFirstValidNonEmptyFilename(diff --git a/backend/Extensions/ProgressExtensions.cs b/backend/Extensions/ProgressExtensions.csnew file mode 100644index 00000000..54bd984a--- /dev/null+++ b/backend/Extensions/ProgressExtensions.cs@@ -0,0 +1,19 @@+namespace NzbWebDAV.Extensions;++public static class ProgressExtensions+{+ public static Progress<int>? ToPercentage(this IProgress<int>? progress, int total)+ {+ return new Progress<int>(x => progress?.Report(x * 100 / total));+ }++ public static Progress<int>? Scale(this IProgress<int>? progress, int numerator, int denominator)+ {+ return new Progress<int>(x => progress?.Report(x * numerator / denominator));+ }++ public static Progress<int>? Offset(this IProgress<int>? progress, int offset)+ {+ return new Progress<int>(x => progress?.Report(x + offset));+ }+}\ No newline at end of filediff --git a/backend/Par2Recovery/Packets/FileDesc.cs b/backend/Par2Recovery/Packets/FileDesc.csnew file mode 100644index 00000000..a6344a2a--- /dev/null+++ b/backend/Par2Recovery/Packets/FileDesc.cs@@ -0,0 +1,109 @@+using System.Text;++namespace NzbWebDAV.Par2Recovery.Packets+{+ public class FileDesc : Par2Packet+ {+ public const string PacketType = "PAR 2.0\0FileDesc";++ public byte[] FileID { get; protected set; }+ public byte[] FileHash { get; protected set; }+ public byte[] File16kHash { get; protected set; }+ public ulong FileLength { get; protected set; }+ public string FileName { get; protected set; }++ public FileDesc(Par2PacketHeader header) : base(header)+ {+ }++ protected override void ParseBody(byte[] body)+ {+ // 16 MD5 Hash The File ID.+ FileID = new byte[16];+ Buffer.BlockCopy(body, 0, FileID, 0, 16);++ // 16 MD5 Hash The MD5 hash of the entire file.+ FileHash = new byte[16];+ Buffer.BlockCopy(body, 16, FileHash, 0, 16);++ // 16 MD5 Hash The MD5-16k. That is, the MD5 hash of the first 16kB of the file.+ File16kHash = new byte[16];+ Buffer.BlockCopy(body, 32, File16kHash, 0, 16);++ // 8 8-byte uint Length of the file.+ FileLength = BitConverter.ToUInt64(body, 48);++ // ?*4 ASCII char array Name of the file. This array is not guaranteed to be null terminated! Subdirectories are indicated by an HTML-style '/' (a.k.a. the UNIX slash). The filename must be unique.+ var nameBuffer = new byte[body.Length - 56];+ Buffer.BlockCopy(body, 56, nameBuffer, 0, nameBuffer.Length);++ // Use UTF8 encoding if it is either ASCII or has valid byte sequences. Otherwise, go with Windows-1252.+ var encoding = IsUTF8(nameBuffer) ? Encoding.UTF8 : Encoding.GetEncoding(1252);+ FileName = encoding.GetString(nameBuffer).Normalize().TrimEnd('\0');+ }++ private static bool IsUTF8(byte[] input)+ {+ if (input == null)+ return false;+ if (input.Length == 0)+ return false;++ // TODO: Check for a BOM.++ for (int i = 0; i < input.Length; i++)+ {+ // Skip low bytes.+ if (input[i] < 0x80)+ continue;++ // Start of 4-byte sequence+ if ((input[i] & 0xF8) == 0xF0)+ {+ if (i + 3 >= input.Length)+ return false; // Invalid sequence length.+ if ((input[i + 1] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ if ((input[i + 2] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ if ((input[i + 3] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ i += 3; // Valid sequence. Skip to the next character.+ continue;+ }++ // Start of 3-byte sequence+ if ((input[i] & 0xF0) == 0xE0)+ {+ if (i + 2 >= input.Length)+ return false; // Invalid sequence length.+ if ((input[i + 1] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ if ((input[i + 2] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ i += 2; // Valid sequence. Skip to the next character.+ continue;+ }++ // Start of 2-byte sequence+ if ((input[i] & 0xE0) == 0xC0)+ {+ if (i + 1 >= input.Length)+ return false; // Invalid sequence length.+ if ((input[i + 1] & 0xC0) != 0x80)+ return false; // Invalid sequence.+ i += 1; // Valid sequence. Skip to the next character.+ continue;+ }+ }++ // Either all high bytes are valid sequences, or there are no high bytes, and US-ASCII is valid UTF-8.+ return true;+ }++ public override string ToString()+ {+ return FileName ?? "FileDesc";+ }+ }+}\ No newline at end of filediff --git a/backend/Par2Recovery/Packets/Par2Packet.cs b/backend/Par2Recovery/Packets/Par2Packet.csnew file mode 100644index 00000000..b90634ea--- /dev/null+++ b/backend/Par2Recovery/Packets/Par2Packet.cs@@ -0,0 +1,35 @@+using System.Runtime.InteropServices;++namespace NzbWebDAV.Par2Recovery.Packets+{+ /// <summary>+ /// Implements the basic Read mechanism, passing the body bytes to any child class.+ /// </summary>+ public class Par2Packet+ {+ public Par2PacketHeader Header { get; protected set; }++ public Par2Packet(Par2PacketHeader header)+ {+ Header = header;+ }++ public async Task ReadAsync(Stream stream)+ {+ // Determine the length of the body as the given packet length, minus the length of the header.+ var bodyLength = Header.PacketLength - (ulong)Marshal.SizeOf<Par2PacketHeader>();++ // Read the calculated number of bytes from the stream.+ var body = new byte[bodyLength];+ await stream.ReadExactlyAsync(body.AsMemory(0, (int)bodyLength));++ // Pass the body to the further implementation for parsing.+ ParseBody(body);+ }++ protected virtual void ParseBody(byte[] body)+ {+ // intentionally left blank+ }+ }+}\ No newline at end of filediff --git a/backend/Par2Recovery/Packets/Par2PacketHeader.cs b/backend/Par2Recovery/Packets/Par2PacketHeader.csnew file mode 100644index 00000000..c3ab1527--- /dev/null+++ b/backend/Par2Recovery/Packets/Par2PacketHeader.cs@@ -0,0 +1,49 @@+using System.Runtime.InteropServices;++namespace NzbWebDAV.Par2Recovery.Packets+{++ /// <summary>+ /// Header structure for binary reading.+ /// </summary>+ [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack = 1)]+ public struct Par2PacketHeader+ {++ /// <summary>+ /// Magic sequence. Used to quickly identify location of packets. Value = {'P', 'A', 'R', '2', '\0', 'P', 'K', 'T'} (ASCII)+ /// </summary>+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]+ public byte[] Magic;++ /// <summary>+ /// Length of the entire packet. Must be multiple of 4. (NB: Includes length of header.)+ /// </summary>+ public UInt64 PacketLength;++ /// <summary>+ /// MD5 Hash of packet. Used as a checksum for the packet.+ /// Calculation starts at first byte of Recovery Set ID and ends at last byte of body.+ /// Does not include the magic sequence, length field or this field.+ /// NB: The MD5 Hash, by its definition, includes the length as if it were appended to the packet.+ /// </summary>+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]+ public byte[] PacketHash;++ /// <summary>+ /// Recovery Set ID. All packets that belong together have the same recovery set ID. (See "main packet" for how it is calculated.)+ /// </summary>+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]+ public byte[] RecoverySetID;++ /// <summary>+ /// Type. Can be anything.+ /// All beginning "PAR " (ASCII) are reserved for specification-defined packets.+ /// Application-specific packets are recommended to begin with the ASCII name of the client.+ /// </summary>+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]+ public byte[] PacketType;++ }++}\ No newline at end of filediff --git a/backend/Par2Recovery/Par2.cs b/backend/Par2Recovery/Par2.csnew file mode 100644index 00000000..c233f872--- /dev/null+++ b/backend/Par2Recovery/Par2.cs@@ -0,0 +1,131 @@+using System.Runtime.InteropServices;+using System.Text;+using System.Text.RegularExpressions;+using NzbWebDAV.Par2Recovery.Packets;+using Serilog;++namespace NzbWebDAV.Par2Recovery+{+ public class Par2+ {+ internal static readonly Regex ParVolume = new(+ @"(.+)\.vol[0-9]{1,10}\+[0-9]{1,10}\.par2$",+ RegexOptions.IgnoreCase+ );++ private const string Par2PacketHeaderMagic = "PAR2\0PKT";++ public static async IAsyncEnumerable<FileDesc> ReadFileDescriptions+ (+ Stream stream,+ CancellationToken ct = default+ )+ {+ Par2Packet? packet = null;+ while (stream.Position < stream.Length && !ct.IsCancellationRequested)+ {+ try+ {+ packet = await ReadPacketAsync(stream);+ }+ catch (Exception e)+ {+ Log.Warning($"Failed to read par2 packet: {e.Message}");+ yield break;+ }++ if (packet is FileDesc newFile)+ {+ yield return newFile;+ }+ }+ }++ private static async Task<Par2Packet> ReadPacketAsync(Stream stream)+ {+ // Read a Packet Header.+ var header = await ReadStructAsync<Par2PacketHeader>(stream);++ // Test if the magic constant matches.+ var magic = Encoding.ASCII.GetString(header.Magic);+ if (!Par2PacketHeaderMagic.Equals(magic))+ throw new ApplicationException("Invalid Magic Constant");++ // Determine which type of packet we have.+ var packetType = Encoding.ASCII.GetString(header.PacketType);+ Par2Packet result;+ switch (packetType)+ {+ case FileDesc.PacketType:+ result = new FileDesc(header);+ break;+ default:+ result = new Par2Packet(header);+ break;+ }++ // Let the packet type parse more of the stream as needed.+ await result.ReadAsync(stream);++ return result;+ }++ /// <summary>+ /// Read a struct as binary from a stream.+ /// </summary>+ /// <typeparam name="T">The struct to read.</typeparam>+ /// <param name="stream">The stream to read from.</param>+ /// <returns>The struct with values read from the stream.</returns>+ private static async Task<T> ReadStructAsync<T>(Stream stream) where T : struct+ {+ var size = Marshal.SizeOf<T>();+ var buffer = new byte[size];+ await stream.ReadExactlyAsync(buffer.AsMemory(0, size)).ConfigureAwait(false);+ var pinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned);+ try+ {+ var structure = Marshal.PtrToStructure<T>(pinnedBuffer.AddrOfPinnedObject());+ return structure;+ }+ finally+ {+ pinnedBuffer.Free();+ }+ }++ private static T ReadStruct<T>(byte[] bytes) where T : struct+ {+ var size = Marshal.SizeOf<T>();+ if (bytes.Length < size)+ {+ throw new ArgumentException("Byte array is too short to represent the struct.", nameof(bytes));+ }++ var pinnedBuffer = GCHandle.Alloc(bytes, GCHandleType.Pinned);+ try+ {+ var structure = Marshal.PtrToStructure<T>(pinnedBuffer.AddrOfPinnedObject());+ return structure;+ }+ finally+ {+ pinnedBuffer.Free();+ }+ }++ public static bool IsPar2Index(byte[] bytes)+ {+ try+ {+ var header = ReadStruct<Par2PacketHeader>(bytes);+ var magic = Encoding.ASCII.GetString(header.Magic);+ return Par2PacketHeaderMagic.Equals(magic)+ && Encoding.ASCII.GetString(header.PacketType) == FileDesc.PacketType;+ }+ catch (Exception e)+ {+ return false;+ }+ }+ }+}\ No newline at end of filediff --git a/backend/Par2Recovery/README.md b/backend/Par2Recovery/README.mdnew file mode 100644index 00000000..4c328f90--- /dev/null+++ b/backend/Par2Recovery/README.md@@ -0,0 +1,28 @@+This directory is derived from the following project:+* https://bitbucket.org/PLN/pln.infra.parreader/src/master/++Some modifications have been made for use by nzbdav++-----++The MIT License (MIT)++Copyright (c) 2013 Peter M. Lemmen (PLN)++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.diff --git a/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.csnew file mode 100644index 00000000..255e8a1d--- /dev/null+++ b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs@@ -0,0 +1,87 @@+// ReSharper disable InconsistentNaming++using NzbWebDAV.Clients;+using NzbWebDAV.Config;+using NzbWebDAV.Exceptions;+using NzbWebDAV.Extensions;+using Usenet.Nzb;+using Usenet.Yenc;++namespace NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;++public static class FetchFirstSegmentsStep+{+ public static async Task<List<NzbFileWithFirstSegment>> FetchFirstSegments+ (+ List<NzbFile> nzbFiles,+ UsenetStreamingClient client,+ ConfigManager configManager,+ CancellationToken cancellationToken = default,+ IProgress<int>? progress = null+ )+ {+ return await nzbFiles+ .Where(x => x.Segments.Count > 0)+ .Select(x => FetchFirstSegment(x, client, cancellationToken))+ .WithConcurrencyAsync(configManager.GetMaxQueueConnections())+ .GetAllAsync(cancellationToken, progress);+ }++ private static async Task<NzbFileWithFirstSegment> FetchFirstSegment+ (+ NzbFile nzbFile,+ UsenetStreamingClient client,+ CancellationToken cancellationToken = default+ )+ {+ try+ {+ // get the first article stream+ var firstSegment = nzbFile.Segments[0].MessageId.Value;+ await using var stream = await client.GetSegmentStreamAsync(firstSegment, cancellationToken);++ // read up to the first 16KB from the stream+ var totalRead = 0;+ var buffer = new byte[16 * 1024];+ while (totalRead < buffer.Length)+ {+ var read = await stream.ReadAsync(buffer.AsMemory(totalRead, buffer.Length - totalRead),+ cancellationToken);+ if (read == 0) break;+ totalRead += read;+ }++ // determine bytes read+ var first16KB = totalRead < buffer.Length+ ? buffer.AsSpan(0, totalRead).ToArray()+ : buffer;++ // return+ return new NzbFileWithFirstSegment+ {+ NzbFile = nzbFile,+ First16KB = first16KB,+ Header = stream.Header,+ MissingFirstSegment = false+ };+ }+ catch (UsenetArticleNotFoundException)+ {+ return new NzbFileWithFirstSegment+ {+ NzbFile = nzbFile,+ First16KB = null,+ Header = null,+ MissingFirstSegment = true+ };+ }+ }++ public class NzbFileWithFirstSegment+ {+ public required NzbFile NzbFile { get; init; }+ public required YencHeader? Header { get; init; }+ public required byte[]? First16KB { get; init; }+ public required bool MissingFirstSegment { get; init; }+ }+}\ No newline at end of filediff --git a/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.csnew file mode 100644index 00000000..ff20518d--- /dev/null+++ b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs@@ -0,0 +1,35 @@+using NzbWebDAV.Clients;+using NzbWebDAV.Par2Recovery;+using NzbWebDAV.Par2Recovery.Packets;+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;++namespace NzbWebDAV.Queue.DeobfuscationSteps._2.GetPar2FileDescriptors;++public static class GetPar2FileDescriptorsStep+{+ public static async Task<List<FileDesc>> GetPar2FileDescriptors+ (+ List<FetchFirstSegmentsStep.NzbFileWithFirstSegment> files,+ UsenetStreamingClient client,+ CancellationToken cancellationToken = default+ )+ {+ // find the par2 index file+ var par2Index = files+ .Where(x => !x.MissingFirstSegment)+ .Where(x => Par2.IsPar2Index(x.First16KB!))+ .MinBy(x => x.NzbFile.Segments.Count);+ if (par2Index is null) return [];++ // return all file descriptors+ var fileDescriptors = new List<FileDesc>();+ var segments = par2Index.NzbFile.Segments.Select(x => x.MessageId.Value).ToArray();+ var filesize = par2Index.NzbFile.Segments.Count == 1+ ? par2Index.Header!.PartOffset + par2Index.Header!.PartSize+ : await client.GetFileSizeAsync(par2Index.NzbFile, cancellationToken);+ await using var stream = client.GetFileStream(segments, filesize, concurrentConnections: 1);+ await foreach (var fileDescriptor in Par2.ReadFileDescriptions(stream, cancellationToken))+ fileDescriptors.Add(fileDescriptor);+ return fileDescriptors;+ }+}\ No newline at end of filediff --git a/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.cs b/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.csnew file mode 100644index 00000000..7ccac502--- /dev/null+++ b/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.cs@@ -0,0 +1,80 @@+using System.Security.Cryptography;+using NzbWebDAV.Extensions;+using NzbWebDAV.Par2Recovery.Packets;+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;+using NzbWebDAV.Utils;+using Usenet.Nzb;++namespace NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;++public static class GetFileInfosStep+{+ public static Dictionary<NzbFile, FileInfo> GetFileInfos+ (+ List<FetchFirstSegmentsStep.NzbFileWithFirstSegment> files,+ List<FileDesc> par2FileDescriptors+ )+ {+ using var md5 = MD5.Create();+ var hashToFileDescMap = GetHashToFileDescMap(par2FileDescriptors);+ var fileinfoDictionary = files.ToDictionary(+ x => x.NzbFile,+ x => GetFileInfo(x, hashToFileDescMap, md5)+ );++ return fileinfoDictionary;+ }++ private static Dictionary<string, FileDesc> GetHashToFileDescMap(List<FileDesc> par2FileDescriptors)+ {+ var hashToFileDescMap = new Dictionary<string, FileDesc>();+ foreach (var descriptor in par2FileDescriptors)+ {+ var hash = BitConverter.ToString(descriptor.File16kHash);+ hashToFileDescMap[hash] = descriptor;+ }++ return hashToFileDescMap;+ }++ private static FileInfo GetFileInfo(+ FetchFirstSegmentsStep.NzbFileWithFirstSegment file,+ Dictionary<string, FileDesc> hashToFilenameMap,+ MD5 md5+ )+ {+ var hash = !file.MissingFirstSegment ? BitConverter.ToString(md5.ComputeHash(file.First16KB!)) : "";+ var fileDesc = hashToFilenameMap.GetValueOrDefault(hash);+ var subjectFileName = file.NzbFile.GetSubjectFileName();+ var headerFileName = file.Header?.FileName ?? "";+ var par2FileName = fileDesc?.FileName ?? "";+ var filename = new List<(string? FileName, int Priority)>+ {+ (FileName: par2FileName, Priority: GetFilenamePriority(par2FileName, 3)),+ (FileName: subjectFileName, Priority: GetFilenamePriority(subjectFileName, 2)),+ (FileName: headerFileName, Priority: GetFilenamePriority(headerFileName, 1)),+ }.Where(x => x.FileName is not null).MaxBy(x => x.Priority).FileName ?? "";++ return new FileInfo()+ {+ FileName = filename,+ FileSize = (long?)fileDesc?.FileLength+ };+ }++ private static int GetFilenamePriority(string? filename, int startingPriority)+ {+ var priority = startingPriority;+ if (string.IsNullOrWhiteSpace(filename)) return priority - 5000;+ if (ObfuscationUtil.IsProbablyObfuscated(filename)) priority -= 1000;+ if (FilenameUtil.IsVideoFile(filename)) priority += 50;+ if (Path.GetExtension(filename).TrimStart('.').Length is >= 2 and <= 4) priority += 10;+ return priority;+ }++ public class FileInfo+ {+ public required string FileName { get; init; }+ public long? FileSize { get; init; }+ }+}\ No newline at end of filediff --git a/backend/Queue/FileProcessors/FileProcessor.cs b/backend/Queue/FileProcessors/FileProcessor.csindex 6e0261cc..53f82360 100644--- a/backend/Queue/FileProcessors/FileProcessor.cs+++ b/backend/Queue/FileProcessors/FileProcessor.cs@@ -1,5 +1,6 @@using NzbWebDAV.Clients;using NzbWebDAV.Exceptions;+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;using NzbWebDAV.Utils;using Serilog;using Usenet.Nzb;@@ -8,15 +9,14 @@ namespace NzbWebDAV.Queue.FileProcessors;public class FileProcessor(NzbFile nzbFile,- string filename,+ GetFileInfosStep.FileInfo fileinfo,UsenetStreamingClient usenet,CancellationToken ct) : BaseProcessor{public static bool CanProcess(string filename){- // skip par2 files- return !filename.EndsWith(".par2", StringComparison.OrdinalIgnoreCase);+ return true;}public override async Task<BaseProcessor.Result?> ProcessAsync()@@ -26,16 +26,16 @@ public class FileProcessor(return new Result(){NzbFile = nzbFile,- FileName = filename,- FileSize = await usenet.GetFileSizeAsync(nzbFile, ct),+ FileName = fileinfo.FileName,+ FileSize = fileinfo.FileSize ?? await usenet.GetFileSizeAsync(nzbFile, ct),};}// Ignore missing articles if it's not a video file.// In that case, simply skip the file altogether.- catch (UsenetArticleNotFoundException) when (!FilenameUtil.IsVideoFile(filename))+ catch (UsenetArticleNotFoundException) when (!FilenameUtil.IsVideoFile(fileinfo.FileName)){- Log.Warning($"File `{filename}` has missing articles. Skipping file since it is not a video.");+ Log.Warning($"File `{fileinfo.FileName}` has missing articles. Skipping file since it is not a video.");return null;}}diff --git a/backend/Queue/FileProcessors/RarProcessor.cs b/backend/Queue/FileProcessors/RarProcessor.csindex c2dbc9b2..a2da65fa 100644--- a/backend/Queue/FileProcessors/RarProcessor.cs+++ b/backend/Queue/FileProcessors/RarProcessor.cs@@ -2,6 +2,9 @@using NzbWebDAV.Clients;using NzbWebDAV.Exceptions;using NzbWebDAV.Extensions;+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;+using NzbWebDAV.Streams;+using NzbWebDAV.Utils;using SharpCompress.Common;using SharpCompress.Common.Rar.Headers;using SharpCompress.IO;@@ -12,21 +15,21 @@ namespace NzbWebDAV.Queue.FileProcessors;public class RarProcessor(NzbFile nzbFile,- string filename,+ GetFileInfosStep.FileInfo fileinfo,UsenetStreamingClient usenet,CancellationToken ct) : BaseProcessor{public static bool CanProcess(string filename){- return IsRarFile(filename);+ return FilenameUtil.IsRarFile(filename);}public override async Task<BaseProcessor.Result?> ProcessAsync(){try{- await using var stream = await usenet.GetFileStream(nzbFile, concurrentConnections: 1, ct);+ await using var stream = await GetNzbFileStream();return new Result(){NzbFile = nzbFile,@@ -48,17 +51,10 @@ public class RarProcessor(}}- private static bool IsRarFile(string? filename)- {- if (string.IsNullOrEmpty(filename)) return false;- return filename.EndsWith(".rar", StringComparison.OrdinalIgnoreCase)- || Regex.IsMatch(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);- }-private string GetArchiveName(){// remove the .rar extension and remove the .partXX if it exists- var sansExtension = Path.GetFileNameWithoutExtension(filename);+ var sansExtension = Path.GetFileNameWithoutExtension(fileinfo.FileName);sansExtension = Regex.Replace(sansExtension, @"\.part\d+$", "");return sansExtension;}@@ -66,11 +62,11 @@ public class RarProcessor(private int GetPartNumber(){// handle the `.partXXX.rar` format- var partMatch = Regex.Match(filename, @"\.part(\d+)\.rar$", RegexOptions.IgnoreCase);+ var partMatch = Regex.Match(fileinfo.FileName, @"\.part(\d+)\.rar$", RegexOptions.IgnoreCase);if (partMatch.Success) return int.Parse(partMatch.Groups[1].Value);// handle the `.rXXX` format- var rMatch = Regex.Match(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);+ var rMatch = Regex.Match(fileinfo.FileName, @"\.r(\d+)$", RegexOptions.IgnoreCase);if (rMatch.Success) return int.Parse(rMatch.Groups[1].Value);// handle the `.rar` format.@@ -102,6 +98,13 @@ public class RarProcessor(return headers;}+ private async Task<NzbFileStream> GetNzbFileStream()+ {+ var segments = nzbFile.Segments.Select(x => x.MessageId.Value).ToArray();+ var filesize = fileinfo.FileSize ?? await usenet.GetFileSizeAsync(nzbFile, ct);+ return usenet.GetFileStream(segments, filesize, concurrentConnections: 1);+ }+public new class Result : BaseProcessor.Result{public NzbFile NzbFile { get; init; } = null!;diff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.csindex ee4982d2..1d932dc3 100644--- a/backend/Queue/QueueItemProcessor.cs+++ b/backend/Queue/QueueItemProcessor.cs@@ -8,6 +8,9 @@ using NzbWebDAV.Database;using NzbWebDAV.Database.Models;using NzbWebDAV.Exceptions;using NzbWebDAV.Extensions;+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;+using NzbWebDAV.Queue.DeobfuscationSteps._2.GetPar2FileDescriptors;+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;using NzbWebDAV.Queue.FileAggregators;using NzbWebDAV.Queue.FileProcessors;using NzbWebDAV.Queue.Validators;@@ -101,22 +104,32 @@ public class QueueItemProcessor(var nzb = await NzbDocument.LoadAsync(stream);var nzbFiles = nzb.Files.Where(x => x.Segments.Count > 0).ToList();- // parse filenames for each nzb file- var filenamesTaskDictionary = nzbFiles.ToDictionary(x => x, x => x.GetFileName(usenetClient));- var filenamesDictionary = new Dictionary<NzbFile, string>();- foreach (var filenameTask in filenamesTaskDictionary)- filenamesDictionary[filenameTask.Key] = await filenameTask.Value;-- // start file processing tasks- var fileProcessingTasks = nzbFiles- .DistinctBy(x => filenamesDictionary[x])- .Select(x => GetFileProcessor(x, filenamesDictionary[x]))+ // part 1 -- get name and size of each nzb file+ var part1Progress = progress+ .Scale(50, 100)+ .ToPercentage(nzbFiles.Count);+ var segments = await FetchFirstSegmentsStep.FetchFirstSegments(+ nzbFiles, usenetClient, configManager, ct, part1Progress);+ var par2FileDescriptors = await GetPar2FileDescriptorsStep.GetPar2FileDescriptors(+ segments, usenetClient, ct);+ var fileInfoDictionary = GetFileInfosStep.GetFileInfos(+ segments, par2FileDescriptors);++ // part 2 -- perform file processing+ var fileProcessors = nzbFiles+ .DistinctBy(x => fileInfoDictionary[x].FileName)+ .Select(x => GetFileProcessor(x, fileInfoDictionary[x])).Where(x => x is not null)- .Select(x => x!.ProcessAsync()).ToList();-- // wait for all file processing tasks to finish- var fileProcessingResults = (await TaskUtil.WhenAllOrError(fileProcessingTasks, progress))+ var part2Progress = progress+ .Offset(50)+ .Scale(50, 100)+ .ToPercentage(fileProcessors.Count);+ var fileProcessingResultsAll = await fileProcessors+ .Select(x => x!.ProcessAsync())+ .WithConcurrencyAsync(configManager.GetMaxQueueConnections())+ .GetAllAsync(ct, part2Progress);+ var fileProcessingResults = fileProcessingResultsAll.Where(x => x is not null).Select(x => x!).ToList();@@ -137,10 +150,10 @@ public class QueueItemProcessor(});}- private BaseProcessor? GetFileProcessor(NzbFile nzbFile, string filename)+ private BaseProcessor? GetFileProcessor(NzbFile nzbFile, GetFileInfosStep.FileInfo fileinfo){- return RarProcessor.CanProcess(filename) ? new RarProcessor(nzbFile, filename, usenetClient, ct)- : FileProcessor.CanProcess(filename) ? new FileProcessor(nzbFile, filename, usenetClient, ct)+ return RarProcessor.CanProcess(fileinfo.FileName) ? new RarProcessor(nzbFile, fileinfo, usenetClient, ct)+ : FileProcessor.CanProcess(fileinfo.FileName) ? new FileProcessor(nzbFile, fileinfo, usenetClient, ct): null;}diff --git a/backend/Streams/NzbFileStream.cs b/backend/Streams/NzbFileStream.csindex 53ee5157..ba17286f 100644--- a/backend/Streams/NzbFileStream.cs+++ b/backend/Streams/NzbFileStream.cs@@ -1,8 +1,6 @@using NzbWebDAV.Clients;using NzbWebDAV.Extensions;using NzbWebDAV.Utils;-using Usenet.Nzb;-using Usenet.Yenc;namespace NzbWebDAV.Streams;@@ -14,34 +12,9 @@ public class NzbFileStream() : Stream{private long _position = 0;- private YencHeaderStream? _firstSegmentStream;private CombinedStream? _innerStream;private bool _disposed;- public YencHeader? FirstYencHeader => _firstSegmentStream?.Header;-- public NzbFileStream- (- NzbFile file,- long fileSize,- INntpClient client,- int concurrentConnections- ) : this(file.GetOrderedSegmentIds(), fileSize, client, concurrentConnections)- {- }--- public NzbFileStream- (- NzbFile file,- YencHeaderStream firstSegmentStream,- INntpClient client,- int concurrentConnections- ) : this(file.GetOrderedSegmentIds(), firstSegmentStream.Header.FileSize, client, concurrentConnections)- {- _firstSegmentStream = firstSegmentStream;- }-public override void Flush(){_innerStream?.Flush();@@ -69,8 +42,6 @@ public class NzbFileStream(_position = absoluteOffset;_innerStream?.Dispose();_innerStream = null;- _firstSegmentStream?.Dispose();- _firstSegmentStream = null;return _position;}@@ -122,16 +93,6 @@ public class NzbFileStream(private CombinedStream GetCombinedStream(int firstSegmentIndex, CancellationToken ct){- if (firstSegmentIndex == 0 && _firstSegmentStream != null)- {- return new CombinedStream(- fileSegmentIds[1..]- .Select(async x => (Stream)await client.GetSegmentStreamAsync(x, ct))- .Prepend(Task.FromResult<Stream>(_firstSegmentStream))- .WithConcurrency(concurrentConnections)- );- }-return new CombinedStream(fileSegmentIds[firstSegmentIndex..].Select(async x => (Stream)await client.GetSegmentStreamAsync(x, ct))@@ -143,7 +104,6 @@ public class NzbFileStream({if (_disposed) return;_innerStream?.Dispose();- _firstSegmentStream?.Dispose();_disposed = true;}@@ -151,7 +111,6 @@ public class NzbFileStream({if (_disposed) return;if (_innerStream != null) await _innerStream.DisposeAsync();- if (_firstSegmentStream != null) await _firstSegmentStream.DisposeAsync();_disposed = true;GC.SuppressFinalize(this);}diff --git a/backend/Utils/FilenameUtil.cs b/backend/Utils/FilenameUtil.csindex 34c5edce..783ce6d3 100644--- a/backend/Utils/FilenameUtil.cs+++ b/backend/Utils/FilenameUtil.cs@@ -1,4 +1,6 @@-namespace NzbWebDAV.Utils;+using System.Text.RegularExpressions;++namespace NzbWebDAV.Utils;public class FilenameUtil{@@ -14,4 +16,11 @@ public class FilenameUtil{return VideoExtensions.Contains(Path.GetExtension(filename).ToLower());}++ public static bool IsRarFile(string? filename)+ {+ if (string.IsNullOrEmpty(filename)) return false;+ return filename.EndsWith(".rar", StringComparison.OrdinalIgnoreCase)+ || Regex.IsMatch(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);+ }}\ No newline at end of filediff --git a/backend/Utils/ObfuscationUtil.cs b/backend/Utils/ObfuscationUtil.csnew file mode 100644index 00000000..77eff2cd--- /dev/null+++ b/backend/Utils/ObfuscationUtil.cs@@ -0,0 +1,87 @@+using System.Text.RegularExpressions;++namespace NzbWebDAV.Utils;++public static class ObfuscationUtil+{+ /// <summary>+ /// Checks if a filename is likely obfuscated.+ /// source: https://github.com/sabnzbd/sabnzbd/blob/64034c5636563b66360aa9dfc1a0b624f4db5cc3/sabnzbd/deobfuscate_filenames.py#L105+ /// </summary>+ /// <param name="filename">The filename or full path to check.</param>+ /// <returns>True if the filename is likely obfuscated, otherwise False.</returns>+ public static bool IsProbablyObfuscated(string filename)+ {+ // Use Path.GetFileNameWithoutExtension for .NET+ var fileBaseName = Path.GetFileNameWithoutExtension(filename);++ // ---+ // First: patterns that are certainly obfuscated++ // ...blabla.H.264/b082fa0beaa644d3aa01045d5b8d0b36.mkv is certainly obfuscated+ // Obfuscated: 32 hex digits+ if (Regex.IsMatch(fileBaseName, @"^[a-f0-9]{32}$"))+ {+ return true;+ }++ // 0675e29e9abfd2.f7d069dab0b853283cc1b069a25f82.6547+ // Obfuscated: starting with 40+ lower case hex digits and/or dots+ if (Regex.IsMatch(fileBaseName, @"^[a-f0-9.]{40,}$"))+ {+ return true;+ }++ // "[BlaBla] something [More] something 5937bc5e32146e.bef89a622e4a23f07b0d3757ad5e8a.a02b264e [Brrr]"+ // So: square brackets plus 30+ hex digit+ if (Regex.IsMatch(fileBaseName, @"[a-f0-9]{30}") && Regex.Matches(fileBaseName, @"\[\w+\]").Count >= 2)+ {+ return true;+ }++ // /some/thing/abc.xyz.a4c567edbcbf27.BLA is certainly obfuscated+ // Obfuscated: starts with 'abc.xyz'+ if (Regex.IsMatch(fileBaseName, @"^abc\.xyz"))+ {+ return true;+ }++ // ---+ // Then: patterns that are not obfuscated but typical, clear names++ // These are signals for the obfuscation versus non-obfuscation+ var decimals = fileBaseName.Count(char.IsDigit);+ var upperChars = fileBaseName.Count(char.IsUpper);+ var lowerChars = fileBaseName.Count(char.IsLower);+ var spacesDots = fileBaseName.Count(c => c is ' ' or '.' or '_'); // space-like symbols++ // Example: "Great Distro"+ if (upperChars >= 2 && lowerChars >= 2 && spacesDots >= 1)+ {+ return false;+ }++ // Example: "this is a download"+ if (spacesDots >= 3)+ {+ return false;+ }++ // Example: "Beast 2020"+ if ((upperChars + lowerChars >= 4) && decimals >= 4 && spacesDots >= 1)+ {+ return false;+ }++ // Example: "Catullus", starts with a capital, and most letters are lower case+ // Not obfuscated: starts with a capital, and most letters are lower case+ if (fileBaseName.Length > 0 && char.IsUpper(fileBaseName[0]) && lowerChars > 2 &&+ (double)upperChars / lowerChars <= 0.25)+ {+ return false;+ }++ // Finally: default to obfuscated+ return true;+ }+}\ No newline at end of file