Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#
5.5 KB
Raw Blame
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using NzbWebDAV.Database;
using NzbWebDAV.Database.Models;
using NzbWebDAV.Websocket;
using Serilog;

namespace NzbWebDAV.Services;

/// <summary>
/// Background service that migrates usenet file data
/// from the sqlite database to the blob-store.
/// </summary>
public class UsenetFileToBlobstoreMigrationService(WebsocketManager websocketManager) : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        Report("Determining number of files to migrate...");
        var initialRemaining = await GetTotalCountLeft(stoppingToken);
        var totalRemaining = initialRemaining;
        ReportProgress(totalRemaining, initialRemaining);
        totalRemaining = await MigrateNzbFiles(totalRemaining, initialRemaining, stoppingToken);
        totalRemaining = await MigrateRarFiles(totalRemaining, initialRemaining, stoppingToken);
        totalRemaining = await MigrateMultipartFiles(totalRemaining, initialRemaining, stoppingToken);
        var complete = initialRemaining - totalRemaining;
        Report(complete == 0
            ? $"Done! Nothing to migrate."
            : $"Done! Migrated {complete}/{initialRemaining} file(s) to the blob-store.");
    }

    private async Task<int> GetTotalCountLeft(CancellationToken ct)
    {
        await using var dbContext = new DavDatabaseContext();
        return await dbContext.NzbFiles.CountAsync(ct) +
               await dbContext.RarFiles.CountAsync(ct) +
               await dbContext.MultipartFiles.CountAsync(ct);
    }

    private Task<int> MigrateNzbFiles(int totalRemaining, int initialRemaining, CancellationToken ct)
    {
        return MigrateUsenetFiles<DavNzbFile>(
            (dbContext) => dbContext.NzbFiles.FirstOrDefaultAsync(ct),
            (file) => file.Id,
            (file) => file.Id = Guid.NewGuid(),
            (dbContext, id) => dbContext.NzbFiles.Remove(new DavNzbFile { Id = id }),
            totalRemaining,
            initialRemaining,
            ct
        );
    }

    private Task<int> MigrateRarFiles(int totalRemaining, int initialRemaining, CancellationToken ct)
    {
        return MigrateUsenetFiles<DavRarFile>(
            (dbContext) => dbContext.RarFiles.FirstOrDefaultAsync(ct),
            (file) => file.Id,
            (file) => file.Id = Guid.NewGuid(),
            (dbContext, id) => dbContext.RarFiles.Remove(new DavRarFile { Id = id }),
            totalRemaining,
            initialRemaining,
            ct
        );
    }

    private Task<int> MigrateMultipartFiles(int totalRemaining, int initialRemaining, CancellationToken ct)
    {
        return MigrateUsenetFiles<DavMultipartFile>(
            (dbContext) => dbContext.MultipartFiles.FirstOrDefaultAsync(ct),
            (file) => file.Id,
            (file) => file.Id = Guid.NewGuid(),
            (dbContext, id) => dbContext.MultipartFiles.Remove(new DavMultipartFile { Id = id }),
            totalRemaining,
            initialRemaining,
            ct
        );
    }

    private async Task<int> MigrateUsenetFiles<T>(
        Func<DavDatabaseContext, Task<T?>> getFileToMigrate,
        Func<T, Guid> getFileToMigrateId,
        Action<T> setFileToMigrateNewId,
        Action<DavDatabaseContext, Guid> removeFileToMigrateFromDb,
        int totalRemaining,
        int initialRemaining,
        CancellationToken ct
    )
    {
        while (!ct.IsCancellationRequested)
        {
            try
            {
                await using var dbContext = new DavDatabaseContext();
                var fileToMigrate = await getFileToMigrate(dbContext);
                if (fileToMigrate == null) return totalRemaining;
                var davItem = await GetDavItem(getFileToMigrateId(fileToMigrate), dbContext, ct);
                dbContext.Entry(fileToMigrate).State = EntityState.Detached;
                setFileToMigrateNewId(fileToMigrate);
                await BlobStore.WriteBlob(getFileToMigrateId(fileToMigrate), fileToMigrate);
                try
                {
                    // database changes
                    davItem.FileBlobId = getFileToMigrateId(fileToMigrate);
                    removeFileToMigrateFromDb(dbContext, davItem.Id);
                    await dbContext.SaveChangesAsync(ct);
                    totalRemaining--;
                    ReportProgress(totalRemaining, initialRemaining);
                }
                catch
                {
                    BlobStore.Delete(getFileToMigrateId(fileToMigrate));
                    throw;
                }
            }
            catch (Exception e)
            {
                Log.Error(e, $"Error migrating usenet-file to blob-store: {e.Message}");
                await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false);
            }
        }

        return totalRemaining;
    }

    private static async Task<DavItem> GetDavItem(Guid id, DavDatabaseContext dbContext, CancellationToken ct)
    {
        return (await dbContext.Items.Where(x => x.Id == id).FirstOrDefaultAsync(ct))
               ?? throw new Exception($"DavItem with id `{id}` not found");
    }

    private void Report(string message)
    {
        _ = websocketManager.SendMessage(WebsocketTopic.UsenetFileToBlobstoreMigrationProgress, message);
    }

    private void ReportProgress(int totalRemaining, int initialRemaining)
    {
        var complete = initialRemaining - totalRemaining;
        Report($"Migrated {complete}/{initialRemaining} file(s) to the blob-store.");
    }
}