Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#

Deobfuscate filenames using par2 files.

nzbdav-dev47d51be09f32
diff --git a/backend/Clients/CachingNntpClient.cs b/backend/Clients/CachingNntpClient.cs
index 59cf7b9c..2f70897b 100644
--- a/backend/Clients/CachingNntpClient.cs
+++ b/backend/Clients/CachingNntpClient.cs
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Caching.Memory;
+using NzbWebDAV.Streams;
using Usenet.Nzb;
using Usenet.Yenc;
@@ -8,21 +9,34 @@ public class CachingNntpClient(INntpClient client, MemoryCache cache) : Wrapping
{
private readonly INntpClient _client = client;
- public override async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)
+ private readonly MemoryCacheEntryOptions _cacheOptions = new()
+ {
+ Size = 1,
+ SlidingExpiration = TimeSpan.FromHours(3)
+ };
+
+ public override async Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, CancellationToken ct)
+ {
+ var cacheKey = segmentId;
+ var stream = await _client.GetSegmentStreamAsync(segmentId, ct);
+ cache.Set(cacheKey, stream.Header, _cacheOptions);
+ return stream;
+ }
+
+ public override async Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken ct)
{
var cacheKey = segmentId;
return (await cache.GetOrCreateAsync(cacheKey, cacheEntry =>
{
- cacheEntry.Size = 1;
- cacheEntry.SlidingExpiration = TimeSpan.FromHours(3);
- return _client.GetSegmentYencHeaderAsync(segmentId, cancellationToken);
+ cacheEntry.SetOptions(_cacheOptions);
+ return _client.GetSegmentYencHeaderAsync(segmentId, ct);
})!)!;
}
- public override async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)
+ public override async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken ct)
{
if (file.Segments.Count == 0) return 0;
- var header = await GetSegmentYencHeaderAsync(file.Segments[0].MessageId, cancellationToken);
- return header.FileSize;
+ var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId, ct);
+ return header.PartOffset + header.PartSize;
}
}
\ No newline at end of file
diff --git a/backend/Clients/ThreadSafeNntpClient.cs b/backend/Clients/ThreadSafeNntpClient.cs
index e36a91c3..95f6c27b 100644
--- a/backend/Clients/ThreadSafeNntpClient.cs
+++ b/backend/Clients/ThreadSafeNntpClient.cs
@@ -77,8 +77,8 @@ public class ThreadSafeNntpClient : INntpClient
public async Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)
{
if (file.Segments.Count == 0) return 0;
- var header = await GetSegmentYencHeaderAsync(file.Segments[0].MessageId.Value, cancellationToken);
- return header.FileSize;
+ var header = await GetSegmentYencHeaderAsync(file.Segments[^1].MessageId.Value, cancellationToken);
+ return header.PartOffset + header.PartSize;
}
public async Task WaitForReady(CancellationToken cancellationToken)
@@ -108,7 +108,7 @@ public class ThreadSafeNntpClient : INntpClient
private IEnumerable<string> GetArticleBody(string segmentId)
{
return _client.Body(new NntpMessageId(segmentId))?.Article?.Body
- ?? throw new UsenetArticleNotFoundException($"Article with message-id {segmentId} not found.");
+ ?? throw new UsenetArticleNotFoundException($"Article with message-id {segmentId} not found.");
}
public void Dispose()
diff --git a/backend/Clients/UsenetStreamingClient.cs b/backend/Clients/UsenetStreamingClient.cs
index fcbedab2..eda26a06 100644
--- a/backend/Clients/UsenetStreamingClient.cs
+++ b/backend/Clients/UsenetStreamingClient.cs
@@ -2,12 +2,10 @@
using NzbWebDAV.Clients.Connections;
using NzbWebDAV.Config;
using NzbWebDAV.Exceptions;
-using NzbWebDAV.Extensions;
using NzbWebDAV.Streams;
using NzbWebDAV.Websocket;
using Usenet.Nntp.Responses;
using Usenet.Nzb;
-using Usenet.Yenc;
namespace NzbWebDAV.Clients;
@@ -86,19 +84,19 @@ public class UsenetStreamingClient
public async Task<NzbFileStream> GetFileStream(NzbFile nzbFile, int concurrentConnections, CancellationToken ct)
{
- var firstSegmentId = nzbFile.GetOrderedSegmentIds().First();
- var firstSegmentStream = await _client.GetSegmentStreamAsync(firstSegmentId, ct);
- return new NzbFileStream(nzbFile, firstSegmentStream, _client, concurrentConnections);
+ var segmentIds = nzbFile.Segments.Select(x => x.MessageId.Value).ToArray();
+ var fileSize = await _client.GetFileSizeAsync(nzbFile, cancellationToken: ct);
+ return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);
}
- public Stream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections)
+ public NzbFileStream GetFileStream(string[] segmentIds, long fileSize, int concurrentConnections)
{
return new NzbFileStream(segmentIds, fileSize, _client, concurrentConnections);
}
- public Task<YencHeader> GetSegmentYencHeaderAsync(string segmentId, CancellationToken cancellationToken)
+ public Task<YencHeaderStream> GetSegmentStreamAsync(string segmentId, CancellationToken cancellationToken)
{
- return _client.GetSegmentYencHeaderAsync(segmentId, cancellationToken);
+ return _client.GetSegmentStreamAsync(segmentId, cancellationToken);
}
public Task<long> GetFileSizeAsync(NzbFile file, CancellationToken cancellationToken)
diff --git a/backend/Extensions/IAsyncEnumerableTaskExtensions.cs b/backend/Extensions/IAsyncEnumerableTaskExtensions.cs
new file mode 100644
index 00000000..f4969e53
--- /dev/null
+++ b/backend/Extensions/IAsyncEnumerableTaskExtensions.cs
@@ -0,0 +1,24 @@
+// ReSharper disable InconsistentNaming
+
+namespace NzbWebDAV.Extensions;
+
+public static class IAsyncEnumerableTaskExtensions
+{
+ public static async Task<List<T>> GetAllAsync<T>
+ (
+ this IAsyncEnumerable<T> asyncEnumerable,
+ CancellationToken ct = default,
+ IProgress<int>? progress = null
+ )
+ {
+ var done = 0;
+ var results = new List<T>();
+ await foreach (var result in asyncEnumerable.WithCancellation(ct))
+ {
+ results.Add(result);
+ progress?.Report(++done);
+ }
+
+ return results;
+ }
+}
\ No newline at end of file
diff --git a/backend/Extensions/IEnumerableTaskExtensions.cs b/backend/Extensions/IEnumerableTaskExtensions.cs
index 0831d711..cf9d3cd9 100644
--- a/backend/Extensions/IEnumerableTaskExtensions.cs
+++ b/backend/Extensions/IEnumerableTaskExtensions.cs
@@ -61,4 +61,31 @@ public static class IEnumerableTaskExtensions
}
}
}
+
+ public static async IAsyncEnumerable<T> WithConcurrencyAsync<T>
+ (
+ this IEnumerable<Task<T>> tasks,
+ int concurrency
+ )
+ {
+ if (concurrency < 1)
+ throw new ArgumentException("concurrency must be greater than zero.");
+
+ var runningTasks = new HashSet<Task<T>>();
+ foreach (var task in tasks)
+ {
+ runningTasks.Add(task);
+ if (runningTasks.Count < concurrency) continue;
+ var completedTask = await Task.WhenAny(runningTasks);
+ runningTasks.Remove(completedTask);
+ yield return await completedTask;
+ }
+
+ while (runningTasks.Count > 0)
+ {
+ var completedTask = await Task.WhenAny(runningTasks);
+ runningTasks.Remove(completedTask);
+ yield return await completedTask;
+ }
+ }
}
\ No newline at end of file
diff --git a/backend/Extensions/NzbFileExtensions.cs b/backend/Extensions/NzbFileExtensions.cs
index 10f11b36..1f01211b 100644
--- a/backend/Extensions/NzbFileExtensions.cs
+++ b/backend/Extensions/NzbFileExtensions.cs
@@ -1,5 +1,4 @@
using System.Text.RegularExpressions;
-using NzbWebDAV.Clients;
using Usenet.Nzb;
namespace NzbWebDAV.Extensions;
@@ -14,38 +13,6 @@ public static class NzbFileExtensions
.ToArray();
}
- public static async Task<string> GetFileName
- (
- this NzbFile file,
- UsenetStreamingClient client,
- CancellationToken ct = default
- )
- {
- // prioritize subject filename with fallback to header filename.
- // unless header filename has a valid extension and subject filename doesn't.
- var subjectFilename = file.GetSubjectFileName();
- if (subjectFilename == "") return await file.GetHeaderFileName(client, ct);
- var subjectFilenameExtension = Path.GetExtension(subjectFilename).TrimStart('.');
- if (subjectFilenameExtension.Length is >= 2 and <= 4) return subjectFilename;
- var headerFilename = await file.GetHeaderFileName(client, ct);
- var headerFilenameExtension = Path.GetExtension(headerFilename).TrimStart('.');
- if (headerFilenameExtension.Length is >=2 and <=4) return headerFilename;
- return subjectFilename;
- }
-
- public static async Task<string> GetHeaderFileName
- (
- this NzbFile file,
- UsenetStreamingClient client,
- CancellationToken ct = default
- )
- {
- if (file.Segments.Count == 0) return "";
- var firstSegment = file.Segments[0].MessageId.Value;
- var header = await client.GetSegmentYencHeaderAsync(firstSegment, default);
- return header.FileName;
- }
-
public static string GetSubjectFileName(this NzbFile file)
{
return GetFirstValidNonEmptyFilename(
diff --git a/backend/Extensions/ProgressExtensions.cs b/backend/Extensions/ProgressExtensions.cs
new file mode 100644
index 00000000..54bd984a
--- /dev/null
+++ b/backend/Extensions/ProgressExtensions.cs
@@ -0,0 +1,19 @@
+namespace NzbWebDAV.Extensions;
+
+public static class ProgressExtensions
+{
+ public static Progress<int>? ToPercentage(this IProgress<int>? progress, int total)
+ {
+ return new Progress<int>(x => progress?.Report(x * 100 / total));
+ }
+
+ public static Progress<int>? Scale(this IProgress<int>? progress, int numerator, int denominator)
+ {
+ return new Progress<int>(x => progress?.Report(x * numerator / denominator));
+ }
+
+ public static Progress<int>? Offset(this IProgress<int>? progress, int offset)
+ {
+ return new Progress<int>(x => progress?.Report(x + offset));
+ }
+}
\ No newline at end of file
diff --git a/backend/Par2Recovery/Packets/FileDesc.cs b/backend/Par2Recovery/Packets/FileDesc.cs
new file mode 100644
index 00000000..a6344a2a
--- /dev/null
+++ b/backend/Par2Recovery/Packets/FileDesc.cs
@@ -0,0 +1,109 @@
+using System.Text;
+
+namespace NzbWebDAV.Par2Recovery.Packets
+{
+ public class FileDesc : Par2Packet
+ {
+ public const string PacketType = "PAR 2.0\0FileDesc";
+
+ public byte[] FileID { get; protected set; }
+ public byte[] FileHash { get; protected set; }
+ public byte[] File16kHash { get; protected set; }
+ public ulong FileLength { get; protected set; }
+ public string FileName { get; protected set; }
+
+ public FileDesc(Par2PacketHeader header) : base(header)
+ {
+ }
+
+ protected override void ParseBody(byte[] body)
+ {
+ // 16 MD5 Hash The File ID.
+ FileID = new byte[16];
+ Buffer.BlockCopy(body, 0, FileID, 0, 16);
+
+ // 16 MD5 Hash The MD5 hash of the entire file.
+ FileHash = new byte[16];
+ Buffer.BlockCopy(body, 16, FileHash, 0, 16);
+
+ // 16 MD5 Hash The MD5-16k. That is, the MD5 hash of the first 16kB of the file.
+ File16kHash = new byte[16];
+ Buffer.BlockCopy(body, 32, File16kHash, 0, 16);
+
+ // 8 8-byte uint Length of the file.
+ FileLength = BitConverter.ToUInt64(body, 48);
+
+ // ?*4 ASCII char array Name of the file. This array is not guaranteed to be null terminated! Subdirectories are indicated by an HTML-style '/' (a.k.a. the UNIX slash). The filename must be unique.
+ var nameBuffer = new byte[body.Length - 56];
+ Buffer.BlockCopy(body, 56, nameBuffer, 0, nameBuffer.Length);
+
+ // Use UTF8 encoding if it is either ASCII or has valid byte sequences. Otherwise, go with Windows-1252.
+ var encoding = IsUTF8(nameBuffer) ? Encoding.UTF8 : Encoding.GetEncoding(1252);
+ FileName = encoding.GetString(nameBuffer).Normalize().TrimEnd('\0');
+ }
+
+ private static bool IsUTF8(byte[] input)
+ {
+ if (input == null)
+ return false;
+ if (input.Length == 0)
+ return false;
+
+ // TODO: Check for a BOM.
+
+ for (int i = 0; i < input.Length; i++)
+ {
+ // Skip low bytes.
+ if (input[i] < 0x80)
+ continue;
+
+ // Start of 4-byte sequence
+ if ((input[i] & 0xF8) == 0xF0)
+ {
+ if (i + 3 >= input.Length)
+ return false; // Invalid sequence length.
+ if ((input[i + 1] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ if ((input[i + 2] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ if ((input[i + 3] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ i += 3; // Valid sequence. Skip to the next character.
+ continue;
+ }
+
+ // Start of 3-byte sequence
+ if ((input[i] & 0xF0) == 0xE0)
+ {
+ if (i + 2 >= input.Length)
+ return false; // Invalid sequence length.
+ if ((input[i + 1] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ if ((input[i + 2] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ i += 2; // Valid sequence. Skip to the next character.
+ continue;
+ }
+
+ // Start of 2-byte sequence
+ if ((input[i] & 0xE0) == 0xC0)
+ {
+ if (i + 1 >= input.Length)
+ return false; // Invalid sequence length.
+ if ((input[i + 1] & 0xC0) != 0x80)
+ return false; // Invalid sequence.
+ i += 1; // Valid sequence. Skip to the next character.
+ continue;
+ }
+ }
+
+ // Either all high bytes are valid sequences, or there are no high bytes, and US-ASCII is valid UTF-8.
+ return true;
+ }
+
+ public override string ToString()
+ {
+ return FileName ?? "FileDesc";
+ }
+ }
+}
\ No newline at end of file
diff --git a/backend/Par2Recovery/Packets/Par2Packet.cs b/backend/Par2Recovery/Packets/Par2Packet.cs
new file mode 100644
index 00000000..b90634ea
--- /dev/null
+++ b/backend/Par2Recovery/Packets/Par2Packet.cs
@@ -0,0 +1,35 @@
+using System.Runtime.InteropServices;
+
+namespace NzbWebDAV.Par2Recovery.Packets
+{
+ /// <summary>
+ /// Implements the basic Read mechanism, passing the body bytes to any child class.
+ /// </summary>
+ public class Par2Packet
+ {
+ public Par2PacketHeader Header { get; protected set; }
+
+ public Par2Packet(Par2PacketHeader header)
+ {
+ Header = header;
+ }
+
+ public async Task ReadAsync(Stream stream)
+ {
+ // Determine the length of the body as the given packet length, minus the length of the header.
+ var bodyLength = Header.PacketLength - (ulong)Marshal.SizeOf<Par2PacketHeader>();
+
+ // Read the calculated number of bytes from the stream.
+ var body = new byte[bodyLength];
+ await stream.ReadExactlyAsync(body.AsMemory(0, (int)bodyLength));
+
+ // Pass the body to the further implementation for parsing.
+ ParseBody(body);
+ }
+
+ protected virtual void ParseBody(byte[] body)
+ {
+ // intentionally left blank
+ }
+ }
+}
\ No newline at end of file
diff --git a/backend/Par2Recovery/Packets/Par2PacketHeader.cs b/backend/Par2Recovery/Packets/Par2PacketHeader.cs
new file mode 100644
index 00000000..c3ab1527
--- /dev/null
+++ b/backend/Par2Recovery/Packets/Par2PacketHeader.cs
@@ -0,0 +1,49 @@
+using System.Runtime.InteropServices;
+
+namespace NzbWebDAV.Par2Recovery.Packets
+{
+
+ /// <summary>
+ /// Header structure for binary reading.
+ /// </summary>
+ [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack = 1)]
+ public struct Par2PacketHeader
+ {
+
+ /// <summary>
+ /// Magic sequence. Used to quickly identify location of packets. Value = {'P', 'A', 'R', '2', '\0', 'P', 'K', 'T'} (ASCII)
+ /// </summary>
+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
+ public byte[] Magic;
+
+ /// <summary>
+ /// Length of the entire packet. Must be multiple of 4. (NB: Includes length of header.)
+ /// </summary>
+ public UInt64 PacketLength;
+
+ /// <summary>
+ /// MD5 Hash of packet. Used as a checksum for the packet.
+ /// Calculation starts at first byte of Recovery Set ID and ends at last byte of body.
+ /// Does not include the magic sequence, length field or this field.
+ /// NB: The MD5 Hash, by its definition, includes the length as if it were appended to the packet.
+ /// </summary>
+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
+ public byte[] PacketHash;
+
+ /// <summary>
+ /// Recovery Set ID. All packets that belong together have the same recovery set ID. (See "main packet" for how it is calculated.)
+ /// </summary>
+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
+ public byte[] RecoverySetID;
+
+ /// <summary>
+ /// Type. Can be anything.
+ /// All beginning "PAR " (ASCII) are reserved for specification-defined packets.
+ /// Application-specific packets are recommended to begin with the ASCII name of the client.
+ /// </summary>
+ [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
+ public byte[] PacketType;
+
+ }
+
+}
\ No newline at end of file
diff --git a/backend/Par2Recovery/Par2.cs b/backend/Par2Recovery/Par2.cs
new file mode 100644
index 00000000..c233f872
--- /dev/null
+++ b/backend/Par2Recovery/Par2.cs
@@ -0,0 +1,131 @@
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using NzbWebDAV.Par2Recovery.Packets;
+using Serilog;
+
+namespace NzbWebDAV.Par2Recovery
+{
+ public class Par2
+ {
+ internal static readonly Regex ParVolume = new(
+ @"(.+)\.vol[0-9]{1,10}\+[0-9]{1,10}\.par2$",
+ RegexOptions.IgnoreCase
+ );
+
+ private const string Par2PacketHeaderMagic = "PAR2\0PKT";
+
+ public static async IAsyncEnumerable<FileDesc> ReadFileDescriptions
+ (
+ Stream stream,
+ CancellationToken ct = default
+ )
+ {
+ Par2Packet? packet = null;
+ while (stream.Position < stream.Length && !ct.IsCancellationRequested)
+ {
+ try
+ {
+ packet = await ReadPacketAsync(stream);
+ }
+ catch (Exception e)
+ {
+ Log.Warning($"Failed to read par2 packet: {e.Message}");
+ yield break;
+ }
+
+ if (packet is FileDesc newFile)
+ {
+ yield return newFile;
+ }
+ }
+ }
+
+ private static async Task<Par2Packet> ReadPacketAsync(Stream stream)
+ {
+ // Read a Packet Header.
+ var header = await ReadStructAsync<Par2PacketHeader>(stream);
+
+ // Test if the magic constant matches.
+ var magic = Encoding.ASCII.GetString(header.Magic);
+ if (!Par2PacketHeaderMagic.Equals(magic))
+ throw new ApplicationException("Invalid Magic Constant");
+
+ // Determine which type of packet we have.
+ var packetType = Encoding.ASCII.GetString(header.PacketType);
+ Par2Packet result;
+ switch (packetType)
+ {
+ case FileDesc.PacketType:
+ result = new FileDesc(header);
+ break;
+ default:
+ result = new Par2Packet(header);
+ break;
+ }
+
+ // Let the packet type parse more of the stream as needed.
+ await result.ReadAsync(stream);
+
+ return result;
+ }
+
+ /// <summary>
+ /// Read a struct as binary from a stream.
+ /// </summary>
+ /// <typeparam name="T">The struct to read.</typeparam>
+ /// <param name="stream">The stream to read from.</param>
+ /// <returns>The struct with values read from the stream.</returns>
+ private static async Task<T> ReadStructAsync<T>(Stream stream) where T : struct
+ {
+ var size = Marshal.SizeOf<T>();
+ var buffer = new byte[size];
+ await stream.ReadExactlyAsync(buffer.AsMemory(0, size)).ConfigureAwait(false);
+ var pinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned);
+ try
+ {
+ var structure = Marshal.PtrToStructure<T>(pinnedBuffer.AddrOfPinnedObject());
+ return structure;
+ }
+ finally
+ {
+ pinnedBuffer.Free();
+ }
+ }
+
+ private static T ReadStruct<T>(byte[] bytes) where T : struct
+ {
+ var size = Marshal.SizeOf<T>();
+ if (bytes.Length < size)
+ {
+ throw new ArgumentException("Byte array is too short to represent the struct.", nameof(bytes));
+ }
+
+ var pinnedBuffer = GCHandle.Alloc(bytes, GCHandleType.Pinned);
+ try
+ {
+ var structure = Marshal.PtrToStructure<T>(pinnedBuffer.AddrOfPinnedObject());
+ return structure;
+ }
+ finally
+ {
+ pinnedBuffer.Free();
+ }
+ }
+
+ public static bool IsPar2Index(byte[] bytes)
+ {
+ try
+ {
+ var header = ReadStruct<Par2PacketHeader>(bytes);
+ var magic = Encoding.ASCII.GetString(header.Magic);
+ return Par2PacketHeaderMagic.Equals(magic)
+ && Encoding.ASCII.GetString(header.PacketType) == FileDesc.PacketType;
+ }
+ catch (Exception e)
+ {
+ return false;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/backend/Par2Recovery/README.md b/backend/Par2Recovery/README.md
new file mode 100644
index 00000000..4c328f90
--- /dev/null
+++ b/backend/Par2Recovery/README.md
@@ -0,0 +1,28 @@
+This directory is derived from the following project:
+* https://bitbucket.org/PLN/pln.infra.parreader/src/master/
+
+Some modifications have been made for use by nzbdav
+
+-----
+
+The MIT License (MIT)
+
+Copyright (c) 2013 Peter M. Lemmen (PLN)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs
new file mode 100644
index 00000000..255e8a1d
--- /dev/null
+++ b/backend/Queue/DeobfuscationSteps/1.FetchFirstSegment/FetchFirstSegmentsStep.cs
@@ -0,0 +1,87 @@
+// ReSharper disable InconsistentNaming
+
+using NzbWebDAV.Clients;
+using NzbWebDAV.Config;
+using NzbWebDAV.Exceptions;
+using NzbWebDAV.Extensions;
+using Usenet.Nzb;
+using Usenet.Yenc;
+
+namespace NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;
+
+public static class FetchFirstSegmentsStep
+{
+ public static async Task<List<NzbFileWithFirstSegment>> FetchFirstSegments
+ (
+ List<NzbFile> nzbFiles,
+ UsenetStreamingClient client,
+ ConfigManager configManager,
+ CancellationToken cancellationToken = default,
+ IProgress<int>? progress = null
+ )
+ {
+ return await nzbFiles
+ .Where(x => x.Segments.Count > 0)
+ .Select(x => FetchFirstSegment(x, client, cancellationToken))
+ .WithConcurrencyAsync(configManager.GetMaxQueueConnections())
+ .GetAllAsync(cancellationToken, progress);
+ }
+
+ private static async Task<NzbFileWithFirstSegment> FetchFirstSegment
+ (
+ NzbFile nzbFile,
+ UsenetStreamingClient client,
+ CancellationToken cancellationToken = default
+ )
+ {
+ try
+ {
+ // get the first article stream
+ var firstSegment = nzbFile.Segments[0].MessageId.Value;
+ await using var stream = await client.GetSegmentStreamAsync(firstSegment, cancellationToken);
+
+ // read up to the first 16KB from the stream
+ var totalRead = 0;
+ var buffer = new byte[16 * 1024];
+ while (totalRead < buffer.Length)
+ {
+ var read = await stream.ReadAsync(buffer.AsMemory(totalRead, buffer.Length - totalRead),
+ cancellationToken);
+ if (read == 0) break;
+ totalRead += read;
+ }
+
+ // determine bytes read
+ var first16KB = totalRead < buffer.Length
+ ? buffer.AsSpan(0, totalRead).ToArray()
+ : buffer;
+
+ // return
+ return new NzbFileWithFirstSegment
+ {
+ NzbFile = nzbFile,
+ First16KB = first16KB,
+ Header = stream.Header,
+ MissingFirstSegment = false
+ };
+ }
+ catch (UsenetArticleNotFoundException)
+ {
+ return new NzbFileWithFirstSegment
+ {
+ NzbFile = nzbFile,
+ First16KB = null,
+ Header = null,
+ MissingFirstSegment = true
+ };
+ }
+ }
+
+ public class NzbFileWithFirstSegment
+ {
+ public required NzbFile NzbFile { get; init; }
+ public required YencHeader? Header { get; init; }
+ public required byte[]? First16KB { get; init; }
+ public required bool MissingFirstSegment { get; init; }
+ }
+}
\ No newline at end of file
diff --git a/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs
new file mode 100644
index 00000000..ff20518d
--- /dev/null
+++ b/backend/Queue/DeobfuscationSteps/2.GetPar2FileDescriptors/GetPar2FileDescriptorsStep.cs
@@ -0,0 +1,35 @@
+using NzbWebDAV.Clients;
+using NzbWebDAV.Par2Recovery;
+using NzbWebDAV.Par2Recovery.Packets;
+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;
+
+namespace NzbWebDAV.Queue.DeobfuscationSteps._2.GetPar2FileDescriptors;
+
+public static class GetPar2FileDescriptorsStep
+{
+ public static async Task<List<FileDesc>> GetPar2FileDescriptors
+ (
+ List<FetchFirstSegmentsStep.NzbFileWithFirstSegment> files,
+ UsenetStreamingClient client,
+ CancellationToken cancellationToken = default
+ )
+ {
+ // find the par2 index file
+ var par2Index = files
+ .Where(x => !x.MissingFirstSegment)
+ .Where(x => Par2.IsPar2Index(x.First16KB!))
+ .MinBy(x => x.NzbFile.Segments.Count);
+ if (par2Index is null) return [];
+
+ // return all file descriptors
+ var fileDescriptors = new List<FileDesc>();
+ var segments = par2Index.NzbFile.Segments.Select(x => x.MessageId.Value).ToArray();
+ var filesize = par2Index.NzbFile.Segments.Count == 1
+ ? par2Index.Header!.PartOffset + par2Index.Header!.PartSize
+ : await client.GetFileSizeAsync(par2Index.NzbFile, cancellationToken);
+ await using var stream = client.GetFileStream(segments, filesize, concurrentConnections: 1);
+ await foreach (var fileDescriptor in Par2.ReadFileDescriptions(stream, cancellationToken))
+ fileDescriptors.Add(fileDescriptor);
+ return fileDescriptors;
+ }
+}
\ No newline at end of file
diff --git a/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.cs b/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.cs
new file mode 100644
index 00000000..7ccac502
--- /dev/null
+++ b/backend/Queue/DeobfuscationSteps/3.GetFileInfos/GetFileInfosStep.cs
@@ -0,0 +1,80 @@
+using System.Security.Cryptography;
+using NzbWebDAV.Extensions;
+using NzbWebDAV.Par2Recovery.Packets;
+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;
+using NzbWebDAV.Utils;
+using Usenet.Nzb;
+
+namespace NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;
+
+public static class GetFileInfosStep
+{
+ public static Dictionary<NzbFile, FileInfo> GetFileInfos
+ (
+ List<FetchFirstSegmentsStep.NzbFileWithFirstSegment> files,
+ List<FileDesc> par2FileDescriptors
+ )
+ {
+ using var md5 = MD5.Create();
+ var hashToFileDescMap = GetHashToFileDescMap(par2FileDescriptors);
+ var fileinfoDictionary = files.ToDictionary(
+ x => x.NzbFile,
+ x => GetFileInfo(x, hashToFileDescMap, md5)
+ );
+
+ return fileinfoDictionary;
+ }
+
+ private static Dictionary<string, FileDesc> GetHashToFileDescMap(List<FileDesc> par2FileDescriptors)
+ {
+ var hashToFileDescMap = new Dictionary<string, FileDesc>();
+ foreach (var descriptor in par2FileDescriptors)
+ {
+ var hash = BitConverter.ToString(descriptor.File16kHash);
+ hashToFileDescMap[hash] = descriptor;
+ }
+
+ return hashToFileDescMap;
+ }
+
+ private static FileInfo GetFileInfo(
+ FetchFirstSegmentsStep.NzbFileWithFirstSegment file,
+ Dictionary<string, FileDesc> hashToFilenameMap,
+ MD5 md5
+ )
+ {
+ var hash = !file.MissingFirstSegment ? BitConverter.ToString(md5.ComputeHash(file.First16KB!)) : "";
+ var fileDesc = hashToFilenameMap.GetValueOrDefault(hash);
+ var subjectFileName = file.NzbFile.GetSubjectFileName();
+ var headerFileName = file.Header?.FileName ?? "";
+ var par2FileName = fileDesc?.FileName ?? "";
+ var filename = new List<(string? FileName, int Priority)>
+ {
+ (FileName: par2FileName, Priority: GetFilenamePriority(par2FileName, 3)),
+ (FileName: subjectFileName, Priority: GetFilenamePriority(subjectFileName, 2)),
+ (FileName: headerFileName, Priority: GetFilenamePriority(headerFileName, 1)),
+ }.Where(x => x.FileName is not null).MaxBy(x => x.Priority).FileName ?? "";
+
+ return new FileInfo()
+ {
+ FileName = filename,
+ FileSize = (long?)fileDesc?.FileLength
+ };
+ }
+
+ private static int GetFilenamePriority(string? filename, int startingPriority)
+ {
+ var priority = startingPriority;
+ if (string.IsNullOrWhiteSpace(filename)) return priority - 5000;
+ if (ObfuscationUtil.IsProbablyObfuscated(filename)) priority -= 1000;
+ if (FilenameUtil.IsVideoFile(filename)) priority += 50;
+ if (Path.GetExtension(filename).TrimStart('.').Length is >= 2 and <= 4) priority += 10;
+ return priority;
+ }
+
+ public class FileInfo
+ {
+ public required string FileName { get; init; }
+ public long? FileSize { get; init; }
+ }
+}
\ No newline at end of file
diff --git a/backend/Queue/FileProcessors/FileProcessor.cs b/backend/Queue/FileProcessors/FileProcessor.cs
index 6e0261cc..53f82360 100644
--- a/backend/Queue/FileProcessors/FileProcessor.cs
+++ b/backend/Queue/FileProcessors/FileProcessor.cs
@@ -1,5 +1,6 @@
using NzbWebDAV.Clients;
using NzbWebDAV.Exceptions;
+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;
using NzbWebDAV.Utils;
using Serilog;
using Usenet.Nzb;
@@ -8,15 +9,14 @@ namespace NzbWebDAV.Queue.FileProcessors;
public class FileProcessor(
NzbFile nzbFile,
- string filename,
+ GetFileInfosStep.FileInfo fileinfo,
UsenetStreamingClient usenet,
CancellationToken ct
) : BaseProcessor
{
public static bool CanProcess(string filename)
{
- // skip par2 files
- return !filename.EndsWith(".par2", StringComparison.OrdinalIgnoreCase);
+ return true;
}
public override async Task<BaseProcessor.Result?> ProcessAsync()
@@ -26,16 +26,16 @@ public class FileProcessor(
return new Result()
{
NzbFile = nzbFile,
- FileName = filename,
- FileSize = await usenet.GetFileSizeAsync(nzbFile, ct),
+ FileName = fileinfo.FileName,
+ FileSize = fileinfo.FileSize ?? await usenet.GetFileSizeAsync(nzbFile, ct),
};
}
// Ignore missing articles if it's not a video file.
// In that case, simply skip the file altogether.
- catch (UsenetArticleNotFoundException) when (!FilenameUtil.IsVideoFile(filename))
+ catch (UsenetArticleNotFoundException) when (!FilenameUtil.IsVideoFile(fileinfo.FileName))
{
- Log.Warning($"File `{filename}` has missing articles. Skipping file since it is not a video.");
+ Log.Warning($"File `{fileinfo.FileName}` has missing articles. Skipping file since it is not a video.");
return null;
}
}
diff --git a/backend/Queue/FileProcessors/RarProcessor.cs b/backend/Queue/FileProcessors/RarProcessor.cs
index c2dbc9b2..a2da65fa 100644
--- a/backend/Queue/FileProcessors/RarProcessor.cs
+++ b/backend/Queue/FileProcessors/RarProcessor.cs
@@ -2,6 +2,9 @@
using NzbWebDAV.Clients;
using NzbWebDAV.Exceptions;
using NzbWebDAV.Extensions;
+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;
+using NzbWebDAV.Streams;
+using NzbWebDAV.Utils;
using SharpCompress.Common;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.IO;
@@ -12,21 +15,21 @@ namespace NzbWebDAV.Queue.FileProcessors;
public class RarProcessor(
NzbFile nzbFile,
- string filename,
+ GetFileInfosStep.FileInfo fileinfo,
UsenetStreamingClient usenet,
CancellationToken ct
) : BaseProcessor
{
public static bool CanProcess(string filename)
{
- return IsRarFile(filename);
+ return FilenameUtil.IsRarFile(filename);
}
public override async Task<BaseProcessor.Result?> ProcessAsync()
{
try
{
- await using var stream = await usenet.GetFileStream(nzbFile, concurrentConnections: 1, ct);
+ await using var stream = await GetNzbFileStream();
return new Result()
{
NzbFile = nzbFile,
@@ -48,17 +51,10 @@ public class RarProcessor(
}
}
- private static bool IsRarFile(string? filename)
- {
- if (string.IsNullOrEmpty(filename)) return false;
- return filename.EndsWith(".rar", StringComparison.OrdinalIgnoreCase)
- || Regex.IsMatch(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);
- }
-
private string GetArchiveName()
{
// remove the .rar extension and remove the .partXX if it exists
- var sansExtension = Path.GetFileNameWithoutExtension(filename);
+ var sansExtension = Path.GetFileNameWithoutExtension(fileinfo.FileName);
sansExtension = Regex.Replace(sansExtension, @"\.part\d+$", "");
return sansExtension;
}
@@ -66,11 +62,11 @@ public class RarProcessor(
private int GetPartNumber()
{
// handle the `.partXXX.rar` format
- var partMatch = Regex.Match(filename, @"\.part(\d+)\.rar$", RegexOptions.IgnoreCase);
+ var partMatch = Regex.Match(fileinfo.FileName, @"\.part(\d+)\.rar$", RegexOptions.IgnoreCase);
if (partMatch.Success) return int.Parse(partMatch.Groups[1].Value);
// handle the `.rXXX` format
- var rMatch = Regex.Match(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);
+ var rMatch = Regex.Match(fileinfo.FileName, @"\.r(\d+)$", RegexOptions.IgnoreCase);
if (rMatch.Success) return int.Parse(rMatch.Groups[1].Value);
// handle the `.rar` format.
@@ -102,6 +98,13 @@ public class RarProcessor(
return headers;
}
+ private async Task<NzbFileStream> GetNzbFileStream()
+ {
+ var segments = nzbFile.Segments.Select(x => x.MessageId.Value).ToArray();
+ var filesize = fileinfo.FileSize ?? await usenet.GetFileSizeAsync(nzbFile, ct);
+ return usenet.GetFileStream(segments, filesize, concurrentConnections: 1);
+ }
+
public new class Result : BaseProcessor.Result
{
public NzbFile NzbFile { get; init; } = null!;
diff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.cs
index ee4982d2..1d932dc3 100644
--- a/backend/Queue/QueueItemProcessor.cs
+++ b/backend/Queue/QueueItemProcessor.cs
@@ -8,6 +8,9 @@ using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
using NzbWebDAV.Exceptions;
using NzbWebDAV.Extensions;
+using NzbWebDAV.Queue.DeobfuscationSteps._1.FetchFirstSegment;
+using NzbWebDAV.Queue.DeobfuscationSteps._2.GetPar2FileDescriptors;
+using NzbWebDAV.Queue.DeobfuscationSteps._3.GetFileInfos;
using NzbWebDAV.Queue.FileAggregators;
using NzbWebDAV.Queue.FileProcessors;
using NzbWebDAV.Queue.Validators;
@@ -101,22 +104,32 @@ public class QueueItemProcessor(
var nzb = await NzbDocument.LoadAsync(stream);
var nzbFiles = nzb.Files.Where(x => x.Segments.Count > 0).ToList();
- // parse filenames for each nzb file
- var filenamesTaskDictionary = nzbFiles.ToDictionary(x => x, x => x.GetFileName(usenetClient));
- var filenamesDictionary = new Dictionary<NzbFile, string>();
- foreach (var filenameTask in filenamesTaskDictionary)
- filenamesDictionary[filenameTask.Key] = await filenameTask.Value;
-
- // start file processing tasks
- var fileProcessingTasks = nzbFiles
- .DistinctBy(x => filenamesDictionary[x])
- .Select(x => GetFileProcessor(x, filenamesDictionary[x]))
+ // part 1 -- get name and size of each nzb file
+ var part1Progress = progress
+ .Scale(50, 100)
+ .ToPercentage(nzbFiles.Count);
+ var segments = await FetchFirstSegmentsStep.FetchFirstSegments(
+ nzbFiles, usenetClient, configManager, ct, part1Progress);
+ var par2FileDescriptors = await GetPar2FileDescriptorsStep.GetPar2FileDescriptors(
+ segments, usenetClient, ct);
+ var fileInfoDictionary = GetFileInfosStep.GetFileInfos(
+ segments, par2FileDescriptors);
+
+ // part 2 -- perform file processing
+ var fileProcessors = nzbFiles
+ .DistinctBy(x => fileInfoDictionary[x].FileName)
+ .Select(x => GetFileProcessor(x, fileInfoDictionary[x]))
.Where(x => x is not null)
- .Select(x => x!.ProcessAsync())
.ToList();
-
- // wait for all file processing tasks to finish
- var fileProcessingResults = (await TaskUtil.WhenAllOrError(fileProcessingTasks, progress))
+ var part2Progress = progress
+ .Offset(50)
+ .Scale(50, 100)
+ .ToPercentage(fileProcessors.Count);
+ var fileProcessingResultsAll = await fileProcessors
+ .Select(x => x!.ProcessAsync())
+ .WithConcurrencyAsync(configManager.GetMaxQueueConnections())
+ .GetAllAsync(ct, part2Progress);
+ var fileProcessingResults = fileProcessingResultsAll
.Where(x => x is not null)
.Select(x => x!)
.ToList();
@@ -137,10 +150,10 @@ public class QueueItemProcessor(
});
}
- private BaseProcessor? GetFileProcessor(NzbFile nzbFile, string filename)
+ private BaseProcessor? GetFileProcessor(NzbFile nzbFile, GetFileInfosStep.FileInfo fileinfo)
{
- return RarProcessor.CanProcess(filename) ? new RarProcessor(nzbFile, filename, usenetClient, ct)
- : FileProcessor.CanProcess(filename) ? new FileProcessor(nzbFile, filename, usenetClient, ct)
+ return RarProcessor.CanProcess(fileinfo.FileName) ? new RarProcessor(nzbFile, fileinfo, usenetClient, ct)
+ : FileProcessor.CanProcess(fileinfo.FileName) ? new FileProcessor(nzbFile, fileinfo, usenetClient, ct)
: null;
}
diff --git a/backend/Streams/NzbFileStream.cs b/backend/Streams/NzbFileStream.cs
index 53ee5157..ba17286f 100644
--- a/backend/Streams/NzbFileStream.cs
+++ b/backend/Streams/NzbFileStream.cs
@@ -1,8 +1,6 @@
using NzbWebDAV.Clients;
using NzbWebDAV.Extensions;
using NzbWebDAV.Utils;
-using Usenet.Nzb;
-using Usenet.Yenc;
namespace NzbWebDAV.Streams;
@@ -14,34 +12,9 @@ public class NzbFileStream(
) : Stream
{
private long _position = 0;
- private YencHeaderStream? _firstSegmentStream;
private CombinedStream? _innerStream;
private bool _disposed;
- public YencHeader? FirstYencHeader => _firstSegmentStream?.Header;
-
- public NzbFileStream
- (
- NzbFile file,
- long fileSize,
- INntpClient client,
- int concurrentConnections
- ) : this(file.GetOrderedSegmentIds(), fileSize, client, concurrentConnections)
- {
- }
-
-
- public NzbFileStream
- (
- NzbFile file,
- YencHeaderStream firstSegmentStream,
- INntpClient client,
- int concurrentConnections
- ) : this(file.GetOrderedSegmentIds(), firstSegmentStream.Header.FileSize, client, concurrentConnections)
- {
- _firstSegmentStream = firstSegmentStream;
- }
-
public override void Flush()
{
_innerStream?.Flush();
@@ -69,8 +42,6 @@ public class NzbFileStream(
_position = absoluteOffset;
_innerStream?.Dispose();
_innerStream = null;
- _firstSegmentStream?.Dispose();
- _firstSegmentStream = null;
return _position;
}
@@ -122,16 +93,6 @@ public class NzbFileStream(
private CombinedStream GetCombinedStream(int firstSegmentIndex, CancellationToken ct)
{
- if (firstSegmentIndex == 0 && _firstSegmentStream != null)
- {
- return new CombinedStream(
- fileSegmentIds[1..]
- .Select(async x => (Stream)await client.GetSegmentStreamAsync(x, ct))
- .Prepend(Task.FromResult<Stream>(_firstSegmentStream))
- .WithConcurrency(concurrentConnections)
- );
- }
-
return new CombinedStream(
fileSegmentIds[firstSegmentIndex..]
.Select(async x => (Stream)await client.GetSegmentStreamAsync(x, ct))
@@ -143,7 +104,6 @@ public class NzbFileStream(
{
if (_disposed) return;
_innerStream?.Dispose();
- _firstSegmentStream?.Dispose();
_disposed = true;
}
@@ -151,7 +111,6 @@ public class NzbFileStream(
{
if (_disposed) return;
if (_innerStream != null) await _innerStream.DisposeAsync();
- if (_firstSegmentStream != null) await _firstSegmentStream.DisposeAsync();
_disposed = true;
GC.SuppressFinalize(this);
}
diff --git a/backend/Utils/FilenameUtil.cs b/backend/Utils/FilenameUtil.cs
index 34c5edce..783ce6d3 100644
--- a/backend/Utils/FilenameUtil.cs
+++ b/backend/Utils/FilenameUtil.cs
@@ -1,4 +1,6 @@
-namespace NzbWebDAV.Utils;
+using System.Text.RegularExpressions;
+
+namespace NzbWebDAV.Utils;
public class FilenameUtil
{
@@ -14,4 +16,11 @@ public class FilenameUtil
{
return VideoExtensions.Contains(Path.GetExtension(filename).ToLower());
}
+
+ public static bool IsRarFile(string? filename)
+ {
+ if (string.IsNullOrEmpty(filename)) return false;
+ return filename.EndsWith(".rar", StringComparison.OrdinalIgnoreCase)
+ || Regex.IsMatch(filename, @"\.r(\d+)$", RegexOptions.IgnoreCase);
+ }
}
\ No newline at end of file
diff --git a/backend/Utils/ObfuscationUtil.cs b/backend/Utils/ObfuscationUtil.cs
new file mode 100644
index 00000000..77eff2cd
--- /dev/null
+++ b/backend/Utils/ObfuscationUtil.cs
@@ -0,0 +1,87 @@
+using System.Text.RegularExpressions;
+
+namespace NzbWebDAV.Utils;
+
+public static class ObfuscationUtil
+{
+ /// <summary>
+ /// Checks if a filename is likely obfuscated.
+ /// source: https://github.com/sabnzbd/sabnzbd/blob/64034c5636563b66360aa9dfc1a0b624f4db5cc3/sabnzbd/deobfuscate_filenames.py#L105
+ /// </summary>
+ /// <param name="filename">The filename or full path to check.</param>
+ /// <returns>True if the filename is likely obfuscated, otherwise False.</returns>
+ public static bool IsProbablyObfuscated(string filename)
+ {
+ // Use Path.GetFileNameWithoutExtension for .NET
+ var fileBaseName = Path.GetFileNameWithoutExtension(filename);
+
+ // ---
+ // First: patterns that are certainly obfuscated
+
+ // ...blabla.H.264/b082fa0beaa644d3aa01045d5b8d0b36.mkv is certainly obfuscated
+ // Obfuscated: 32 hex digits
+ if (Regex.IsMatch(fileBaseName, @"^[a-f0-9]{32}$"))
+ {
+ return true;
+ }
+
+ // 0675e29e9abfd2.f7d069dab0b853283cc1b069a25f82.6547
+ // Obfuscated: starting with 40+ lower case hex digits and/or dots
+ if (Regex.IsMatch(fileBaseName, @"^[a-f0-9.]{40,}$"))
+ {
+ return true;
+ }
+
+ // "[BlaBla] something [More] something 5937bc5e32146e.bef89a622e4a23f07b0d3757ad5e8a.a02b264e [Brrr]"
+ // So: square brackets plus 30+ hex digit
+ if (Regex.IsMatch(fileBaseName, @"[a-f0-9]{30}") && Regex.Matches(fileBaseName, @"\[\w+\]").Count >= 2)
+ {
+ return true;
+ }
+
+ // /some/thing/abc.xyz.a4c567edbcbf27.BLA is certainly obfuscated
+ // Obfuscated: starts with 'abc.xyz'
+ if (Regex.IsMatch(fileBaseName, @"^abc\.xyz"))
+ {
+ return true;
+ }
+
+ // ---
+ // Then: patterns that are not obfuscated but typical, clear names
+
+ // These are signals for the obfuscation versus non-obfuscation
+ var decimals = fileBaseName.Count(char.IsDigit);
+ var upperChars = fileBaseName.Count(char.IsUpper);
+ var lowerChars = fileBaseName.Count(char.IsLower);
+ var spacesDots = fileBaseName.Count(c => c is ' ' or '.' or '_'); // space-like symbols
+
+ // Example: "Great Distro"
+ if (upperChars >= 2 && lowerChars >= 2 && spacesDots >= 1)
+ {
+ return false;
+ }
+
+ // Example: "this is a download"
+ if (spacesDots >= 3)
+ {
+ return false;
+ }
+
+ // Example: "Beast 2020"
+ if ((upperChars + lowerChars >= 4) && decimals >= 4 && spacesDots >= 1)
+ {
+ return false;
+ }
+
+ // Example: "Catullus", starts with a capital, and most letters are lower case
+ // Not obfuscated: starts with a capital, and most letters are lower case
+ if (fileBaseName.Length > 0 && char.IsUpper(fileBaseName[0]) && lowerChars > 2 &&
+ (double)upperChars / lowerChars <= 0.25)
+ {
+ return false;
+ }
+
+ // Finally: default to obfuscated
+ return true;
+ }
+}
\ No newline at end of file