Usenet streaming with a WebDAV server and a SABnzbd-compatible API
C#
Write usenet stream data to blobstore instead of db.
nzbdav-dev678afde5f41f
diff --git a/backend/Database/DavDatabaseContext.cs b/backend/Database/DavDatabaseContext.csindex 6b44cf97..d17608e1 100644--- a/backend/Database/DavDatabaseContext.cs+++ b/backend/Database/DavDatabaseContext.cs@@ -12,8 +12,8 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value)public static string ConfigPath => EnvironmentUtil.GetEnvironmentVariable("CONFIG_PATH") ?? "/config";public static string DatabaseFilePath => Path.Join(ConfigPath, "db.sqlite");- private static readonly Lazy<DbContextOptions<DavDatabaseContext>> Options = new(- () => new DbContextOptionsBuilder<DavDatabaseContext>()+ private static readonly Lazy<DbContextOptions<DavDatabaseContext>> Options = new(() =>+ new DbContextOptionsBuilder<DavDatabaseContext>().UseSqlite($"Data Source={DatabaseFilePath}").AddInterceptors(new SqliteForeignKeyEnabler()).Options@@ -33,6 +33,11 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value)public DbSet<ConfigItem> ConfigItems => Set<ConfigItem>();public DbSet<BlobCleanupItem> BlobCleanupItems => Set<BlobCleanupItem>();+ // blob items+ public List<DavNzbFile> BlobNzbFiles = [];+ public List<DavRarFile> BlobRarFiles = [];+ public List<DavMultipartFile> BlobMultipartFiles = [];+// tablesprotected override void OnModelCreating(ModelBuilder b){@@ -426,4 +431,50 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value).ValueGeneratedNever();});}++ public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = new CancellationToken())+ {+ try+ {+ // save blobs to blob-store+ foreach (var blobNzbFile in BlobNzbFiles)+ await BlobStore.WriteBlob(blobNzbFile.Id, blobNzbFile);+ foreach (var blobRarFile in BlobRarFiles)+ await BlobStore.WriteBlob(blobRarFile.Id, blobRarFile);+ foreach (var blobMultipartFile in BlobMultipartFiles)+ await BlobStore.WriteBlob(blobMultipartFile.Id, blobMultipartFile);++ // save db changes+ var result = await base.SaveChangesAsync(cancellationToken);++ // clear pending blob writes+ BlobNzbFiles.Clear();+ BlobRarFiles.Clear();+ BlobMultipartFiles.Clear();++ // return+ return result;+ }+ catch+ {+ // on errors, remove any already-written blob files+ foreach (var blobNzbFile in BlobNzbFiles)+ BlobStore.Delete(blobNzbFile.Id);+ foreach (var blobRarFile in BlobRarFiles)+ BlobStore.Delete(blobRarFile.Id);+ foreach (var blobMultipartFile in BlobMultipartFiles)+ BlobStore.Delete(blobMultipartFile.Id);++ // rethrow the exception+ throw;+ }+ }++ public void ClearChangeTracker()+ {+ ChangeTracker.Clear();+ BlobNzbFiles.Clear();+ BlobRarFiles.Clear();+ BlobMultipartFiles.Clear();+ }}\ No newline at end of filediff --git a/backend/Database/Models/DavItem.cs b/backend/Database/Models/DavItem.csindex 66384550..684169e6 100644--- a/backend/Database/Models/DavItem.cs+++ b/backend/Database/Models/DavItem.cs@@ -30,7 +30,8 @@ public class DavItemItemType type,ItemSubType subType,DateTimeOffset? releaseDate,- DateTimeOffset? lastHealthCheck+ DateTimeOffset? lastHealthCheck,+ Guid? fileBlobId){return new DavItem()@@ -48,7 +49,8 @@ public class DavItemLastHealthCheck = lastHealthCheck,NextHealthCheck = releaseDate != null && lastHealthCheck != null? releaseDate.Value + 2 * (lastHealthCheck.Value - releaseDate.Value)- : null+ : null,+ FileBlobId = fileBlobId};}diff --git a/backend/Queue/FileAggregators/BaseAggregator.cs b/backend/Queue/FileAggregators/BaseAggregator.csindex 00e8e812..f1f5c39a 100644--- a/backend/Queue/FileAggregators/BaseAggregator.cs+++ b/backend/Queue/FileAggregators/BaseAggregator.cs@@ -51,7 +51,8 @@ public abstract class BaseAggregatortype: DavItem.ItemType.Directory,subType: DavItem.ItemSubType.Directory,releaseDate: null,- lastHealthCheck: null+ lastHealthCheck: null,+ fileBlobId: null);_directoryCache.Add(pathKey, directory);DBClient.Ctx.Items.Add(directory);diff --git a/backend/Queue/FileAggregators/FileAggregator.cs b/backend/Queue/FileAggregators/FileAggregator.csindex c1d3d4e7..fe565542 100644--- a/backend/Queue/FileAggregators/FileAggregator.cs+++ b/backend/Queue/FileAggregators/FileAggregator.cs@@ -19,6 +19,12 @@ public class FileAggregator(DavDatabaseClient dbClient, DavItem mountDirectory,var parentDirectory = EnsureParentDirectory(result.FileName);var name = Path.GetFileName(result.FileName);+ var davNzbFile = new DavNzbFile()+ {+ Id = Guid.NewGuid(),+ SegmentIds = result.NzbFile.GetSegmentIds(),+ };+var davItem = DavItem.New(id: Guid.NewGuid(),parent: parentDirectory,@@ -27,17 +33,12 @@ public class FileAggregator(DavDatabaseClient dbClient, DavItem mountDirectory,type: DavItem.ItemType.UsenetFile,subType: DavItem.ItemSubType.NzbFile,releaseDate: result.ReleaseDate,- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,+ fileBlobId: davNzbFile.Id);- var davNzbFile = new DavNzbFile()- {- Id = davItem.Id,- SegmentIds = result.NzbFile.GetSegmentIds(),- };-dbClient.Ctx.Items.Add(davItem);- dbClient.Ctx.NzbFiles.Add(davNzbFile);+ dbClient.Ctx.BlobNzbFiles.Add(davNzbFile);}}}\ No newline at end of filediff --git a/backend/Queue/FileAggregators/MultipartMkvAggregator.cs b/backend/Queue/FileAggregators/MultipartMkvAggregator.csindex 9388f4f4..15dc5b6a 100644--- a/backend/Queue/FileAggregators/MultipartMkvAggregator.cs+++ b/backend/Queue/FileAggregators/MultipartMkvAggregator.cs@@ -4,8 +4,7 @@ using NzbWebDAV.Queue.FileProcessors;namespace NzbWebDAV.Queue.FileAggregators;-public class MultipartMkvAggregator-(+public class MultipartMkvAggregator(DavDatabaseClient dbClient,DavItem mountDirectory,bool checkedFullHealth@@ -31,6 +30,15 @@ public class MultipartMkvAggregatorvar parentDirectory = MountDirectory;var name = multipartMkvFile.Filename;+ var davMultipartFile = new DavMultipartFile()+ {+ Id = Guid.NewGuid(),+ Metadata = new DavMultipartFile.Meta()+ {+ FileParts = fileParts.ToArray()+ }+ };+var davItem = DavItem.New(id: Guid.NewGuid(),parent: parentDirectory,@@ -39,20 +47,12 @@ public class MultipartMkvAggregatortype: DavItem.ItemType.UsenetFile,subType: DavItem.ItemSubType.MultipartFile,releaseDate: multipartMkvFile.ReleaseDate,- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,+ fileBlobId: davMultipartFile.Id);- var davMultipartFile = new DavMultipartFile()- {- Id = davItem.Id,- Metadata = new DavMultipartFile.Meta()- {- FileParts = fileParts.ToArray()- }- };-dbClient.Ctx.Items.Add(davItem);- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);}}}\ No newline at end of filediff --git a/backend/Queue/FileAggregators/RarAggregator.cs b/backend/Queue/FileAggregators/RarAggregator.csindex 54ff3a55..9620cd9e 100644--- a/backend/Queue/FileAggregators/RarAggregator.cs+++ b/backend/Queue/FileAggregators/RarAggregator.cs@@ -52,20 +52,9 @@ public class RarAggregator(DavDatabaseClient dbClient, DavItem mountDirectory, bif (archiveFiles.Count == 1 && ObfuscationUtil.IsProbablyObfuscated(name))name = mountDirectory.Name + Path.GetExtension(name);- var davItem = DavItem.New(- id: Guid.NewGuid(),- parent: parentDirectory,- name: name,- fileSize: fileSize,- type: DavItem.ItemType.UsenetFile,- subType: DavItem.ItemSubType.MultipartFile,- releaseDate: fileParts.First().ReleaseDate,- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null- );-var davMultipartFile = new DavMultipartFile(){- Id = davItem.Id,+ Id = Guid.NewGuid(),Metadata = new DavMultipartFile.Meta(){AesParams = aesParams,@@ -78,8 +67,20 @@ public class RarAggregator(DavDatabaseClient dbClient, DavItem mountDirectory, b}};+ var davItem = DavItem.New(+ id: Guid.NewGuid(),+ parent: parentDirectory,+ name: name,+ fileSize: fileSize,+ type: DavItem.ItemType.UsenetFile,+ subType: DavItem.ItemSubType.MultipartFile,+ releaseDate: fileParts.First().ReleaseDate,+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,+ fileBlobId: davMultipartFile.Id+ );+dbClient.Ctx.Items.Add(davItem);- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);}}diff --git a/backend/Queue/FileAggregators/SevenZipAggregator.cs b/backend/Queue/FileAggregators/SevenZipAggregator.csindex 734a3ea7..00a5f381 100644--- a/backend/Queue/FileAggregators/SevenZipAggregator.cs+++ b/backend/Queue/FileAggregators/SevenZipAggregator.cs@@ -38,6 +38,12 @@ public class SevenZipAggregator(if (sevenZipFiles.Count == 1 && ObfuscationUtil.IsProbablyObfuscated(name))name = mountDirectory.Name + Path.GetExtension(name);+ var davMultipartFile = new DavMultipartFile()+ {+ Id = Guid.NewGuid(),+ Metadata = davMultipartFileMeta+ };+var davItem = DavItem.New(id: Guid.NewGuid(),parent: parentDirectory,@@ -47,17 +53,12 @@ public class SevenZipAggregator(type: DavItem.ItemType.UsenetFile,subType: DavItem.ItemSubType.MultipartFile,releaseDate: sevenZipFile.ReleaseDate,- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,+ fileBlobId: davMultipartFile.Id);- var davMultipartFile = new DavMultipartFile()- {- Id = davItem.Id,- Metadata = davMultipartFileMeta- };-dbClient.Ctx.Items.Add(davItem);- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);}}}\ No newline at end of filediff --git a/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs b/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.csindex 16849f31..0c00e162 100644--- a/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs+++ b/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs@@ -41,29 +41,35 @@ public class BlocklistedFilePostProcessor(ConfigManager configManager, DavDataba{if (davItem.SubType == DavItem.ItemSubType.NzbFile){+ dbClient.Ctx.BlobNzbFiles.RemoveAll(x => x.Id == davItem.FileBlobId);var file = dbClient.Ctx.ChangeTracker.Entries<DavNzbFile>().Where(x => x.State == EntityState.Added).Select(x => x.Entity)- .First(x => x.Id == davItem.Id);- dbClient.Ctx.NzbFiles.Remove(file);+ .FirstOrDefault(x => x.Id == davItem.Id);+ if (file is not null)+ dbClient.Ctx.NzbFiles.Remove(file);}else if (davItem.SubType == DavItem.ItemSubType.RarFile){+ dbClient.Ctx.BlobRarFiles.RemoveAll(x => x.Id == davItem.FileBlobId);var file = dbClient.Ctx.ChangeTracker.Entries<DavRarFile>().Where(x => x.State == EntityState.Added).Select(x => x.Entity)- .First(x => x.Id == davItem.Id);- dbClient.Ctx.RarFiles.Remove(file);+ .FirstOrDefault(x => x.Id == davItem.Id);+ if (file is not null)+ dbClient.Ctx.RarFiles.Remove(file);}else if (davItem.SubType == DavItem.ItemSubType.MultipartFile){+ dbClient.Ctx.BlobMultipartFiles.RemoveAll(x => x.Id == davItem.FileBlobId);var file = dbClient.Ctx.ChangeTracker.Entries<DavMultipartFile>().Where(x => x.State == EntityState.Added).Select(x => x.Entity)- .First(x => x.Id == davItem.Id);- dbClient.Ctx.MultipartFiles.Remove(file);+ .FirstOrDefault(x => x.Id == davItem.Id);+ if (file is not null)+ dbClient.Ctx.MultipartFiles.Remove(file);}elsediff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.csindex 08b2de9a..009cfc9c 100644--- a/backend/Queue/QueueItemProcessor.cs+++ b/backend/Queue/QueueItemProcessor.cs@@ -49,7 +49,7 @@ public class QueueItemProcessor(catch (Exception e) when (e.GetBaseException().IsCancellationException()){Log.Information($"Processing of queue item `{queueItem.JobName}` was cancelled.");- dbClient.Ctx.ChangeTracker.Clear();+ dbClient.Ctx.ClearChangeTracker();}// when a retryable error is encountered@@ -61,7 +61,7 @@ public class QueueItemProcessor(try{Log.Error($"Failed to process job, `{queueItem.JobName}` -- {e.Message}");- dbClient.Ctx.ChangeTracker.Clear();+ dbClient.Ctx.ClearChangeTracker();queueItem.PauseUntil = DateTime.Now.AddMinutes(1);dbClient.Ctx.QueueItems.Attach(queueItem);dbClient.Ctx.Entry(queueItem).Property(x => x.PauseUntil).IsModified = true;@@ -260,7 +260,8 @@ public class QueueItemProcessor(type: DavItem.ItemType.Directory,subType: DavItem.ItemSubType.Directory,releaseDate: null,- lastHealthCheck: null+ lastHealthCheck: null,+ fileBlobId: null);dbClient.Ctx.Items.Add(categoryFolder);return categoryFolder;@@ -284,7 +285,8 @@ public class QueueItemProcessor(type: DavItem.ItemType.Directory,subType: DavItem.ItemSubType.Directory,releaseDate: null,- lastHealthCheck: null+ lastHealthCheck: null,+ fileBlobId: null);dbClient.Ctx.Items.Add(mountFolder);return Task.FromResult(mountFolder);@@ -307,7 +309,8 @@ public class QueueItemProcessor(type: DavItem.ItemType.Directory,subType: DavItem.ItemSubType.Directory,releaseDate: null,- lastHealthCheck: null+ lastHealthCheck: null,+ fileBlobId: null);dbClient.Ctx.Items.Add(mountFolder);return mountFolder;@@ -342,7 +345,7 @@ public class QueueItemProcessor(Func<Task<DavItem?>>? databaseOperations = null){- dbClient.Ctx.ChangeTracker.Clear();+ dbClient.Ctx.ClearChangeTracker();var mountFolder = databaseOperations != null ? await databaseOperations.Invoke().ConfigureAwait(false) : null;var historyItem = CreateHistoryItem(mountFolder, startTime, error);var historySlot = GetHistoryResponse.HistorySlot.FromHistoryItem(historyItem, mountFolder, configManager);diff --git a/backend/WebDav/DatabaseStoreWatchFolder.cs b/backend/WebDav/DatabaseStoreWatchFolder.csindex d772d1ff..e6fac088 100644--- a/backend/WebDav/DatabaseStoreWatchFolder.cs+++ b/backend/WebDav/DatabaseStoreWatchFolder.cs@@ -85,7 +85,7 @@ public class DatabaseStoreWatchFolder(return DavStatusCode.NotFound;// delete the item- dbClient.Ctx.ChangeTracker.Clear();+ dbClient.Ctx.ClearChangeTracker();await controller.RemoveFromQueue(new RemoveFromQueueRequest(){NzoIds = [item.Id],