Skip to content

Usenet streaming with a WebDAV server and a SABnzbd-compatible API

C#

Write usenet stream data to blobstore instead of db.

nzbdav-dev678afde5f41f
diff --git a/backend/Database/DavDatabaseContext.cs b/backend/Database/DavDatabaseContext.cs
index 6b44cf97..d17608e1 100644
--- a/backend/Database/DavDatabaseContext.cs
+++ b/backend/Database/DavDatabaseContext.cs
@@ -12,8 +12,8 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value)
public static string ConfigPath => EnvironmentUtil.GetEnvironmentVariable("CONFIG_PATH") ?? "/config";
public static string DatabaseFilePath => Path.Join(ConfigPath, "db.sqlite");
- private static readonly Lazy<DbContextOptions<DavDatabaseContext>> Options = new(
- () => new DbContextOptionsBuilder<DavDatabaseContext>()
+ private static readonly Lazy<DbContextOptions<DavDatabaseContext>> Options = new(() =>
+ new DbContextOptionsBuilder<DavDatabaseContext>()
.UseSqlite($"Data Source={DatabaseFilePath}")
.AddInterceptors(new SqliteForeignKeyEnabler())
.Options
@@ -33,6 +33,11 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value)
public DbSet<ConfigItem> ConfigItems => Set<ConfigItem>();
public DbSet<BlobCleanupItem> BlobCleanupItems => Set<BlobCleanupItem>();
+ // blob items
+ public List<DavNzbFile> BlobNzbFiles = [];
+ public List<DavRarFile> BlobRarFiles = [];
+ public List<DavMultipartFile> BlobMultipartFiles = [];
+
// tables
protected override void OnModelCreating(ModelBuilder b)
{
@@ -426,4 +431,50 @@ public sealed class DavDatabaseContext() : DbContext(Options.Value)
.ValueGeneratedNever();
});
}
+
+ public override async Task<int> SaveChangesAsync(CancellationToken cancellationToken = new CancellationToken())
+ {
+ try
+ {
+ // save blobs to blob-store
+ foreach (var blobNzbFile in BlobNzbFiles)
+ await BlobStore.WriteBlob(blobNzbFile.Id, blobNzbFile);
+ foreach (var blobRarFile in BlobRarFiles)
+ await BlobStore.WriteBlob(blobRarFile.Id, blobRarFile);
+ foreach (var blobMultipartFile in BlobMultipartFiles)
+ await BlobStore.WriteBlob(blobMultipartFile.Id, blobMultipartFile);
+
+ // save db changes
+ var result = await base.SaveChangesAsync(cancellationToken);
+
+ // clear pending blob writes
+ BlobNzbFiles.Clear();
+ BlobRarFiles.Clear();
+ BlobMultipartFiles.Clear();
+
+ // return
+ return result;
+ }
+ catch
+ {
+ // on errors, remove any already-written blob files
+ foreach (var blobNzbFile in BlobNzbFiles)
+ BlobStore.Delete(blobNzbFile.Id);
+ foreach (var blobRarFile in BlobRarFiles)
+ BlobStore.Delete(blobRarFile.Id);
+ foreach (var blobMultipartFile in BlobMultipartFiles)
+ BlobStore.Delete(blobMultipartFile.Id);
+
+ // rethrow the exception
+ throw;
+ }
+ }
+
+ public void ClearChangeTracker()
+ {
+ ChangeTracker.Clear();
+ BlobNzbFiles.Clear();
+ BlobRarFiles.Clear();
+ BlobMultipartFiles.Clear();
+ }
}
\ No newline at end of file
diff --git a/backend/Database/Models/DavItem.cs b/backend/Database/Models/DavItem.cs
index 66384550..684169e6 100644
--- a/backend/Database/Models/DavItem.cs
+++ b/backend/Database/Models/DavItem.cs
@@ -30,7 +30,8 @@ public class DavItem
ItemType type,
ItemSubType subType,
DateTimeOffset? releaseDate,
- DateTimeOffset? lastHealthCheck
+ DateTimeOffset? lastHealthCheck,
+ Guid? fileBlobId
)
{
return new DavItem()
@@ -48,7 +49,8 @@ public class DavItem
LastHealthCheck = lastHealthCheck,
NextHealthCheck = releaseDate != null && lastHealthCheck != null
? releaseDate.Value + 2 * (lastHealthCheck.Value - releaseDate.Value)
- : null
+ : null,
+ FileBlobId = fileBlobId
};
}
diff --git a/backend/Queue/FileAggregators/BaseAggregator.cs b/backend/Queue/FileAggregators/BaseAggregator.cs
index 00e8e812..f1f5c39a 100644
--- a/backend/Queue/FileAggregators/BaseAggregator.cs
+++ b/backend/Queue/FileAggregators/BaseAggregator.cs
@@ -51,7 +51,8 @@ public abstract class BaseAggregator
type: DavItem.ItemType.Directory,
subType: DavItem.ItemSubType.Directory,
releaseDate: null,
- lastHealthCheck: null
+ lastHealthCheck: null,
+ fileBlobId: null
);
_directoryCache.Add(pathKey, directory);
DBClient.Ctx.Items.Add(directory);
diff --git a/backend/Queue/FileAggregators/FileAggregator.cs b/backend/Queue/FileAggregators/FileAggregator.cs
index c1d3d4e7..fe565542 100644
--- a/backend/Queue/FileAggregators/FileAggregator.cs
+++ b/backend/Queue/FileAggregators/FileAggregator.cs
@@ -19,6 +19,12 @@ public class FileAggregator(DavDatabaseClient dbClient, DavItem mountDirectory,
var parentDirectory = EnsureParentDirectory(result.FileName);
var name = Path.GetFileName(result.FileName);
+ var davNzbFile = new DavNzbFile()
+ {
+ Id = Guid.NewGuid(),
+ SegmentIds = result.NzbFile.GetSegmentIds(),
+ };
+
var davItem = DavItem.New(
id: Guid.NewGuid(),
parent: parentDirectory,
@@ -27,17 +33,12 @@ public class FileAggregator(DavDatabaseClient dbClient, DavItem mountDirectory,
type: DavItem.ItemType.UsenetFile,
subType: DavItem.ItemSubType.NzbFile,
releaseDate: result.ReleaseDate,
- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null
+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,
+ fileBlobId: davNzbFile.Id
);
- var davNzbFile = new DavNzbFile()
- {
- Id = davItem.Id,
- SegmentIds = result.NzbFile.GetSegmentIds(),
- };
-
dbClient.Ctx.Items.Add(davItem);
- dbClient.Ctx.NzbFiles.Add(davNzbFile);
+ dbClient.Ctx.BlobNzbFiles.Add(davNzbFile);
}
}
}
\ No newline at end of file
diff --git a/backend/Queue/FileAggregators/MultipartMkvAggregator.cs b/backend/Queue/FileAggregators/MultipartMkvAggregator.cs
index 9388f4f4..15dc5b6a 100644
--- a/backend/Queue/FileAggregators/MultipartMkvAggregator.cs
+++ b/backend/Queue/FileAggregators/MultipartMkvAggregator.cs
@@ -4,8 +4,7 @@ using NzbWebDAV.Queue.FileProcessors;
namespace NzbWebDAV.Queue.FileAggregators;
-public class MultipartMkvAggregator
-(
+public class MultipartMkvAggregator(
DavDatabaseClient dbClient,
DavItem mountDirectory,
bool checkedFullHealth
@@ -31,6 +30,15 @@ public class MultipartMkvAggregator
var parentDirectory = MountDirectory;
var name = multipartMkvFile.Filename;
+ var davMultipartFile = new DavMultipartFile()
+ {
+ Id = Guid.NewGuid(),
+ Metadata = new DavMultipartFile.Meta()
+ {
+ FileParts = fileParts.ToArray()
+ }
+ };
+
var davItem = DavItem.New(
id: Guid.NewGuid(),
parent: parentDirectory,
@@ -39,20 +47,12 @@ public class MultipartMkvAggregator
type: DavItem.ItemType.UsenetFile,
subType: DavItem.ItemSubType.MultipartFile,
releaseDate: multipartMkvFile.ReleaseDate,
- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null
+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,
+ fileBlobId: davMultipartFile.Id
);
- var davMultipartFile = new DavMultipartFile()
- {
- Id = davItem.Id,
- Metadata = new DavMultipartFile.Meta()
- {
- FileParts = fileParts.ToArray()
- }
- };
-
dbClient.Ctx.Items.Add(davItem);
- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);
+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);
}
}
}
\ No newline at end of file
diff --git a/backend/Queue/FileAggregators/RarAggregator.cs b/backend/Queue/FileAggregators/RarAggregator.cs
index 54ff3a55..9620cd9e 100644
--- a/backend/Queue/FileAggregators/RarAggregator.cs
+++ b/backend/Queue/FileAggregators/RarAggregator.cs
@@ -52,20 +52,9 @@ public class RarAggregator(DavDatabaseClient dbClient, DavItem mountDirectory, b
if (archiveFiles.Count == 1 && ObfuscationUtil.IsProbablyObfuscated(name))
name = mountDirectory.Name + Path.GetExtension(name);
- var davItem = DavItem.New(
- id: Guid.NewGuid(),
- parent: parentDirectory,
- name: name,
- fileSize: fileSize,
- type: DavItem.ItemType.UsenetFile,
- subType: DavItem.ItemSubType.MultipartFile,
- releaseDate: fileParts.First().ReleaseDate,
- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null
- );
-
var davMultipartFile = new DavMultipartFile()
{
- Id = davItem.Id,
+ Id = Guid.NewGuid(),
Metadata = new DavMultipartFile.Meta()
{
AesParams = aesParams,
@@ -78,8 +67,20 @@ public class RarAggregator(DavDatabaseClient dbClient, DavItem mountDirectory, b
}
};
+ var davItem = DavItem.New(
+ id: Guid.NewGuid(),
+ parent: parentDirectory,
+ name: name,
+ fileSize: fileSize,
+ type: DavItem.ItemType.UsenetFile,
+ subType: DavItem.ItemSubType.MultipartFile,
+ releaseDate: fileParts.First().ReleaseDate,
+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,
+ fileBlobId: davMultipartFile.Id
+ );
+
dbClient.Ctx.Items.Add(davItem);
- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);
+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);
}
}
diff --git a/backend/Queue/FileAggregators/SevenZipAggregator.cs b/backend/Queue/FileAggregators/SevenZipAggregator.cs
index 734a3ea7..00a5f381 100644
--- a/backend/Queue/FileAggregators/SevenZipAggregator.cs
+++ b/backend/Queue/FileAggregators/SevenZipAggregator.cs
@@ -38,6 +38,12 @@ public class SevenZipAggregator(
if (sevenZipFiles.Count == 1 && ObfuscationUtil.IsProbablyObfuscated(name))
name = mountDirectory.Name + Path.GetExtension(name);
+ var davMultipartFile = new DavMultipartFile()
+ {
+ Id = Guid.NewGuid(),
+ Metadata = davMultipartFileMeta
+ };
+
var davItem = DavItem.New(
id: Guid.NewGuid(),
parent: parentDirectory,
@@ -47,17 +53,12 @@ public class SevenZipAggregator(
type: DavItem.ItemType.UsenetFile,
subType: DavItem.ItemSubType.MultipartFile,
releaseDate: sevenZipFile.ReleaseDate,
- lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null
+ lastHealthCheck: checkedFullHealth ? DateTimeOffset.UtcNow : null,
+ fileBlobId: davMultipartFile.Id
);
- var davMultipartFile = new DavMultipartFile()
- {
- Id = davItem.Id,
- Metadata = davMultipartFileMeta
- };
-
dbClient.Ctx.Items.Add(davItem);
- dbClient.Ctx.MultipartFiles.Add(davMultipartFile);
+ dbClient.Ctx.BlobMultipartFiles.Add(davMultipartFile);
}
}
}
\ No newline at end of file
diff --git a/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs b/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs
index 16849f31..0c00e162 100644
--- a/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs
+++ b/backend/Queue/PostProcessors/BlocklistedFilePostProcessor.cs
@@ -41,29 +41,35 @@ public class BlocklistedFilePostProcessor(ConfigManager configManager, DavDataba
{
if (davItem.SubType == DavItem.ItemSubType.NzbFile)
{
+ dbClient.Ctx.BlobNzbFiles.RemoveAll(x => x.Id == davItem.FileBlobId);
var file = dbClient.Ctx.ChangeTracker.Entries<DavNzbFile>()
.Where(x => x.State == EntityState.Added)
.Select(x => x.Entity)
- .First(x => x.Id == davItem.Id);
- dbClient.Ctx.NzbFiles.Remove(file);
+ .FirstOrDefault(x => x.Id == davItem.Id);
+ if (file is not null)
+ dbClient.Ctx.NzbFiles.Remove(file);
}
else if (davItem.SubType == DavItem.ItemSubType.RarFile)
{
+ dbClient.Ctx.BlobRarFiles.RemoveAll(x => x.Id == davItem.FileBlobId);
var file = dbClient.Ctx.ChangeTracker.Entries<DavRarFile>()
.Where(x => x.State == EntityState.Added)
.Select(x => x.Entity)
- .First(x => x.Id == davItem.Id);
- dbClient.Ctx.RarFiles.Remove(file);
+ .FirstOrDefault(x => x.Id == davItem.Id);
+ if (file is not null)
+ dbClient.Ctx.RarFiles.Remove(file);
}
else if (davItem.SubType == DavItem.ItemSubType.MultipartFile)
{
+ dbClient.Ctx.BlobMultipartFiles.RemoveAll(x => x.Id == davItem.FileBlobId);
var file = dbClient.Ctx.ChangeTracker.Entries<DavMultipartFile>()
.Where(x => x.State == EntityState.Added)
.Select(x => x.Entity)
- .First(x => x.Id == davItem.Id);
- dbClient.Ctx.MultipartFiles.Remove(file);
+ .FirstOrDefault(x => x.Id == davItem.Id);
+ if (file is not null)
+ dbClient.Ctx.MultipartFiles.Remove(file);
}
else
diff --git a/backend/Queue/QueueItemProcessor.cs b/backend/Queue/QueueItemProcessor.cs
index 08b2de9a..009cfc9c 100644
--- a/backend/Queue/QueueItemProcessor.cs
+++ b/backend/Queue/QueueItemProcessor.cs
@@ -49,7 +49,7 @@ public class QueueItemProcessor(
catch (Exception e) when (e.GetBaseException().IsCancellationException())
{
Log.Information($"Processing of queue item `{queueItem.JobName}` was cancelled.");
- dbClient.Ctx.ChangeTracker.Clear();
+ dbClient.Ctx.ClearChangeTracker();
}
// when a retryable error is encountered
@@ -61,7 +61,7 @@ public class QueueItemProcessor(
try
{
Log.Error($"Failed to process job, `{queueItem.JobName}` -- {e.Message}");
- dbClient.Ctx.ChangeTracker.Clear();
+ dbClient.Ctx.ClearChangeTracker();
queueItem.PauseUntil = DateTime.Now.AddMinutes(1);
dbClient.Ctx.QueueItems.Attach(queueItem);
dbClient.Ctx.Entry(queueItem).Property(x => x.PauseUntil).IsModified = true;
@@ -260,7 +260,8 @@ public class QueueItemProcessor(
type: DavItem.ItemType.Directory,
subType: DavItem.ItemSubType.Directory,
releaseDate: null,
- lastHealthCheck: null
+ lastHealthCheck: null,
+ fileBlobId: null
);
dbClient.Ctx.Items.Add(categoryFolder);
return categoryFolder;
@@ -284,7 +285,8 @@ public class QueueItemProcessor(
type: DavItem.ItemType.Directory,
subType: DavItem.ItemSubType.Directory,
releaseDate: null,
- lastHealthCheck: null
+ lastHealthCheck: null,
+ fileBlobId: null
);
dbClient.Ctx.Items.Add(mountFolder);
return Task.FromResult(mountFolder);
@@ -307,7 +309,8 @@ public class QueueItemProcessor(
type: DavItem.ItemType.Directory,
subType: DavItem.ItemSubType.Directory,
releaseDate: null,
- lastHealthCheck: null
+ lastHealthCheck: null,
+ fileBlobId: null
);
dbClient.Ctx.Items.Add(mountFolder);
return mountFolder;
@@ -342,7 +345,7 @@ public class QueueItemProcessor(
Func<Task<DavItem?>>? databaseOperations = null
)
{
- dbClient.Ctx.ChangeTracker.Clear();
+ dbClient.Ctx.ClearChangeTracker();
var mountFolder = databaseOperations != null ? await databaseOperations.Invoke().ConfigureAwait(false) : null;
var historyItem = CreateHistoryItem(mountFolder, startTime, error);
var historySlot = GetHistoryResponse.HistorySlot.FromHistoryItem(historyItem, mountFolder, configManager);
diff --git a/backend/WebDav/DatabaseStoreWatchFolder.cs b/backend/WebDav/DatabaseStoreWatchFolder.cs
index d772d1ff..e6fac088 100644
--- a/backend/WebDav/DatabaseStoreWatchFolder.cs
+++ b/backend/WebDav/DatabaseStoreWatchFolder.cs
@@ -85,7 +85,7 @@ public class DatabaseStoreWatchFolder(
return DavStatusCode.NotFound;
// delete the item
- dbClient.Ctx.ChangeTracker.Clear();
+ dbClient.Ctx.ClearChangeTracker();
await controller.RemoveFromQueue(new RemoveFromQueueRequest()
{
NzoIds = [item.Id],