Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#
5.8 KB
Raw Blame
using System.Diagnostics.CodeAnalysis;
using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
using NzbWebDAV.Extensions;
using NzbWebDAV.Models;
using NzbWebDAV.Queue.FileProcessors;
using NzbWebDAV.Utils;

namespace NzbWebDAV.Queue.FileAggregators;

public class RarAggregator(DavDatabaseClient dbClient, DavItem mountDirectory, bool checkedFullHealth) : BaseAggregator
{
    protected override DavDatabaseClient DBClient => dbClient;
    protected override DavItem MountDirectory => mountDirectory;

    public override void UpdateDatabase(List<BaseProcessor.Result> processorResults)
    {
        var fileSegments = processorResults
            .OfType<RarProcessor.Result>()
            .SelectMany(x => x.StoredFileSegments)
            .ToList();

        ProcessArchive(fileSegments);
    }

    private void ProcessArchive(List<RarProcessor.StoredFileSegment> fileSegments)
    {
        var archiveFiles = new Dictionary<string, List<RarProcessor.StoredFileSegment>>();
        foreach (var fileSegment in fileSegments)
        {
            if (!archiveFiles.ContainsKey(fileSegment.PathWithinArchive))
                archiveFiles.Add(fileSegment.PathWithinArchive, []);

            archiveFiles[fileSegment.PathWithinArchive].Add(fileSegment);
        }

        foreach (var archiveFile in archiveFiles)
        {
            // Ensure we have all volumes necessary for this file.
            ValidateVolumes(archiveFile.Value);

            // Initialize dav-item fields
            var pathWithinArchive = archiveFile.Key;
            var fileParts = SortByPartNumber(archiveFile.Value);
            var aesParams = fileParts.Select(x => x.AesParams).FirstOrDefault(x => x != null);
            var fileSize = aesParams?.DecodedSize ?? fileParts.Sum(x => x.ByteRangeWithinPart.Count);
            var parentDirectory = EnsureParentDirectory(pathWithinArchive);
            var name = Path.GetFileName(pathWithinArchive);

            // If there is only one file in the archive and the file-name is obfuscated,
            // then rename the file to the same name as the containing mount directory.
            if (archiveFiles.Count == 1 && ObfuscationUtil.IsProbablyObfuscated(name))
                name = mountDirectory.Name + Path.GetExtension(name);

            var davMultipartFile = new DavMultipartFile()
            {
                Id = Guid.NewGuid(),
                Metadata = new DavMultipartFile.Meta()
                {
                    AesParams = aesParams,
                    FileParts = fileParts.Select(x => new DavMultipartFile.FilePart()
                    {
                        SegmentIds = x.NzbFile.GetSegmentIds(),
                        SegmentIdByteRange = LongRange.FromStartAndSize(0, x.PartSize),
                        FilePartByteRange = x.ByteRangeWithinPart
                    }).ToArray(),
                }
            };

            var davItem = DavItem.New(
                id: Guid.NewGuid(),
                parent: parentDirectory,
                name: name,
                fileSize: fileSize,
                type: DavItem.ItemType.UsenetFile,
                subType: DavItem.ItemSubType.MultipartFile,
                releaseDate: fileParts.First().ReleaseDate,
                lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,
                historyItemId: MountDirectory.HistoryItemId,
                fileBlobId: davMultipartFile.Id,
                nzbBlobId: MountDirectory.HistoryItemId
            );

            dbClient.Ctx.Items.Add(davItem);
            dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);
        }
    }

    private static RarProcessor.StoredFileSegment[] SortByPartNumber(
        List<RarProcessor.StoredFileSegment> storedFileSegments)
    {
        // Find delta between part number from headers and filenames.
        var delta = storedFileSegments
            .Select(x => x.PartNumber)
            .Where(x => x is { PartNumberFromHeader: >= 0, PartNumberFromFilename: >= 0 })
            .Select(x => x.PartNumberFromHeader - x.PartNumberFromFilename)
            .GroupBy(x => x)
            .MaxBy(x => x.Count())?.Key;

        // Ensure there are no duplicate part numbers.
        var allPartNumbers = storedFileSegments.Select(x => GetNormalizedPartNumber(x.PartNumber, delta));
        ValidatePartNumbers(allPartNumbers);

        // Sort by part numbers and return.
        return storedFileSegments
            .OrderBy(x => GetNormalizedPartNumber(x.PartNumber, delta))
            .ToArray();
    }

    private static void ValidateVolumes(List<RarProcessor.StoredFileSegment> storedFileSegments)
    {
        if (storedFileSegments.Count == 0) return;
        var distinctUncompressedSizes = storedFileSegments.Select(x => x.FileUncompressedSize).Distinct().ToList();
        if (distinctUncompressedSizes.Count != 1)
            throw new InvalidDataException("Inconsistent rar file size detected.");
        var expected = distinctUncompressedSizes[0];
        var actual = storedFileSegments.Sum(x => x.ByteRangeWithinPart.Count);
        if (Math.Abs(actual - expected) > 16)
            throw new InvalidDataException("Missing rar volumes detected.");
    }

    [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")]
    private static void ValidatePartNumbers(IEnumerable<int> partNumbers)
    {
        var count = partNumbers.Count();
        var uniqueCount = partNumbers.Distinct().Count();
        if (count != uniqueCount)
            throw new InvalidDataException("Rar archive has duplicate volume numbers.");
    }

    private static int GetNormalizedPartNumber(RarProcessor.PartNumber partNumber, int? delta)
    {
        if (partNumber.PartNumberFromHeader >= 0) return partNumber.PartNumberFromHeader!.Value;
        if (partNumber.PartNumberFromFilename >= 0) return partNumber.PartNumberFromFilename!.Value + (delta ?? 0);
        return -1;
    }
}