Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

Undo a failed conversion only where the path was its alone

KnifeLemonf250beaef50a
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 7cdd35a9..4a7c9e40 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -45,6 +45,9 @@
own (`DirectOnly`), never a step of a longer one.
5. **Run** – `JobQueue` runs steps with per-engine concurrency limits. Intermediate files live in a temp folder;
final outputs are named by `OutputPathResolver` (pattern, folder, conflict policy; the source is never overwritten).
+ Each name stays reserved while a conversion holds it. A conversion that fails or is cancelled deletes what it began
+ writing, but only at a path that was its alone: nothing was there before and no other conversion was handed the
+ same name ("Overwrite"). It frees its names and removes an output folder it made if that is left empty.
"Merge into one PDF" converts each file to PDF and merges with PDFsharp.
"Compress into one archive" packs the dropped files, unconverted, with the archive engine (`IFileCombiner`).
6. **Report** – progress and results appear in the toast window and in Home → Recent conversions.
diff --git a/src/Filee.Core/Conversion/JobQueue.cs b/src/Filee.Core/Conversion/JobQueue.cs
index a5404653..99bddd8b 100644
--- a/src/Filee.Core/Conversion/JobQueue.cs
+++ b/src/Filee.Core/Conversion/JobQueue.cs
@@ -20,7 +20,13 @@ public sealed class JobQueue : IAsyncDisposable
private readonly ILogger _log;
private readonly Channel<ConversionJob> _channel = Channel.CreateUnbounded<ConversionJob>();
private readonly ConcurrentDictionary<string, SemaphoreSlim> _engineGates = new();
- private readonly HashSet<string> _reservedOutputs = new(StringComparer.OrdinalIgnoreCase);
+ /// <summary>
+ /// Output paths handed out and not released yet, with how many conversions hold each. Also the lock for everything
+ /// about output paths.
+ /// </summary>
+ private readonly Dictionary<string, Reservation> _reservedOutputs = new(StringComparer.OrdinalIgnoreCase);
+ /// <summary>Output folders this queue made (a subfolder, a fixed folder that wasn't there): removed again when left empty.</summary>
+ private readonly HashSet<string> _madeFolders = new(Platform.FileSystemPaths.Comparer);
private readonly Task _worker;
private int _busy;
@@ -122,8 +128,12 @@ public sealed class JobQueue : IAsyncDisposable
finally
{
lock (_reservedOutputs)
+ {
foreach (var output in job.Outputs)
- _reservedOutputs.Remove(output);
+ Unreserve(output);
+ // Folders no running job writes into any more are the user's from now on.
+ _madeFolders.RemoveWhere(folder => !HasReservationIn(folder));
+ }
TryDelete(workDir);
Release(job);
}
@@ -573,7 +583,7 @@ public sealed class JobQueue : IAsyncDisposable
inputs = PackZip(job, file, index, inputs, ct);
// One output, and nothing else handed out for it (a Markdown file's picture folder goes with it).
else if (finalOutput && inputs.Count == 1 && final is { Handed: 1 })
- inputs = [NextToSource(job, file, index, inputs[0])];
+ inputs = [NextToSource(job, file, index, inputs[0], final)];
// A name the engine took but wrote nothing to is free again.
final?.Release(except: inputs);
@@ -718,7 +728,7 @@ public sealed class JobQueue : IAsyncDisposable
/// the subfolder if that left it empty. A file that can't be moved (the policy skips an existing file, or the file
/// is in use) stays in the subfolder.
/// </summary>
- private string NextToSource(ConversionJob job, FileResult file, int index, string output)
+ private string NextToSource(ConversionJob job, FileResult file, int index, string output, FinalAllocator allocator)
{
var rule = ForOneFile(job.Preset.Output);
if (ReferenceEquals(rule, job.Preset.Output) || !File.Exists(output)
@@ -733,7 +743,7 @@ public sealed class JobQueue : IAsyncDisposable
{
var tokens = new OutputPathResolver.Tokens(file.SourcePath, job.PresetDisplayName, index, job.CreatedAt);
var target = OutputPathResolver.Resolve(rule, tokens, Path.GetExtension(output),
- p => !PathsEqual(p, output) && (File.Exists(p) || Directory.Exists(p) || _reservedOutputs.Contains(p)));
+ p => !PathsEqual(p, output) && (File.Exists(p) || Directory.Exists(p) || _reservedOutputs.ContainsKey(p)));
if (target is null || PathsEqual(target, output))
return output;
try
@@ -746,20 +756,9 @@ public sealed class JobQueue : IAsyncDisposable
_log.LogWarning(ex, "Could not move {Output} next to its source", output);
return output;
}
- _reservedOutputs.Remove(output);
- _reservedOutputs.Add(target);
-
- // Another file of this or a running job may still be written into the same subfolder.
- var subfolder = Path.GetDirectoryName(output)!;
- if (!_reservedOutputs.Any(p => PathsEqual(Path.GetDirectoryName(p) ?? "", subfolder)))
- {
- try
- {
- if (!Directory.EnumerateFileSystemEntries(subfolder).Any())
- Directory.Delete(subfolder);
- }
- catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { }
- }
+ allocator.Moved(output, target);
+ // Another file of this or a running job may still be written into the same subfolder: RemoveIfEmpty waits.
+ RemoveIfEmpty(Path.GetDirectoryName(output)!);
return target;
}
@@ -767,6 +766,59 @@ public sealed class JobQueue : IAsyncDisposable
string.Equals(Path.GetFullPath(a), Path.GetFullPath(b), Platform.FileSystemPaths.Comparison);
}
+ /// <summary>How many conversions hold an output path, and whether more than one ever did.</summary>
+ private sealed class Reservation
+ {
+ public int Holders = 1;
+
+ /// <summary>
+ /// Handed to a second conversion too: "Overwrite" gives two files that become the same name the same path. Then
+ /// neither may delete it when it fails: the other one's finished file may be there.
+ /// </summary>
+ public bool Shared;
+ }
+
+ /// <summary>Takes <paramref name="path"/> (under the lock); true if nobody held it.</summary>
+ private bool Reserve(string path)
+ {
+ if (_reservedOutputs.TryGetValue(path, out var reservation))
+ {
+ reservation.Holders++;
+ reservation.Shared = true;
+ return false;
+ }
+ _reservedOutputs[path] = new Reservation();
+ return true;
+ }
+
+ /// <summary>Lets go of <paramref name="path"/> (under the lock): free once nobody holds it.</summary>
+ private void Unreserve(string path)
+ {
+ if (_reservedOutputs.TryGetValue(path, out var reservation) && --reservation.Holders <= 0)
+ _reservedOutputs.Remove(path);
+ }
+
+ private bool HasReservationIn(string folder) =>
+ _reservedOutputs.Keys.Any(p => Platform.FileSystemPaths.Comparer.Equals(Path.GetDirectoryName(Path.GetFullPath(p)), folder));
+
+ /// <summary>
+ /// Removes a folder this queue made for outputs once nothing is in it and no running conversion is to write into it
+ /// (under the lock). Never a folder that was there before, and never one with anything inside.
+ /// </summary>
+ private void RemoveIfEmpty(string folder)
+ {
+ folder = Path.GetFullPath(folder);
+ if (!_madeFolders.Contains(folder) || HasReservationIn(folder))
+ return;
+ try
+ {
+ if (Directory.Exists(folder) && !Directory.EnumerateFileSystemEntries(folder).Any())
+ Directory.Delete(folder);
+ _madeFolders.Remove(folder);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { }
+ }
+
private void Notify(ConversionJob job)
{
try
@@ -822,8 +874,11 @@ public sealed class JobQueue : IAsyncDisposable
/// <summary>Allocates final output paths according to the preset's <see cref="OutputRule"/>.</summary>
private sealed class FinalAllocator(JobQueue queue, OutputRule rule, OutputPathResolver.Tokens tokens) : IOutputAllocator
{
- /// <summary>Every path handed out, and whether nothing was there before (not a file it overwrites).</summary>
- private readonly List<(string Path, bool New)> _allocated = [];
+ /// <summary>
+ /// Every path handed out, and whether it is this conversion's alone: nothing was there before (not a file it
+ /// overwrites) and no other conversion had it.
+ /// </summary>
+ private readonly List<(string Path, bool Own)> _allocated = [];
/// <summary>How many paths were handed out: an output and the picture folder that goes with it are two.</summary>
public int Handed => _allocated.Count;
@@ -834,19 +889,42 @@ public sealed class JobQueue : IAsyncDisposable
lock (queue._reservedOutputs)
{
var path = OutputPathResolver.Resolve(rule, tokens, extension,
- p => File.Exists(p) || Directory.Exists(p) || queue._reservedOutputs.Contains(p), suffix);
+ p => File.Exists(p) || Directory.Exists(p) || queue._reservedOutputs.ContainsKey(p), suffix);
if (path is null)
return null;
- _allocated.Add((path, !File.Exists(path) && !Directory.Exists(path)));
- queue._reservedOutputs.Add(path);
- Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+ var nothingThere = !File.Exists(path) && !Directory.Exists(path);
+ _allocated.Add((path, queue.Reserve(path) && nothingThere));
+ var folder = Path.GetFullPath(Path.GetDirectoryName(path)!);
+ if (!Directory.Exists(folder))
+ {
+ Directory.CreateDirectory(folder);
+ queue._madeFolders.Add(folder);
+ }
// An empty extension asks for a folder (archive extraction): the converter creates it.
return path;
}
}
- /// <summary>Frees the names handed out but not in <paramref name="except"/> (the outputs the engine returned).</summary>
+ /// <summary>The one output was moved to <paramref name="to"/> (next to its source): its name goes with it.</summary>
+ public void Moved(string from, string to)
+ {
+ lock (queue._reservedOutputs)
+ {
+ var index = _allocated.FindIndex(a => string.Equals(a.Path, from, StringComparison.OrdinalIgnoreCase));
+ if (index < 0)
+ return;
+ // Finished and in its place: nothing to undo there any more.
+ _allocated[index] = (to, false);
+ queue.Unreserve(from);
+ queue.Reserve(to);
+ }
+ }
+
+ /// <summary>
+ /// Frees the names handed out but not in <paramref name="except"/> (the outputs the engine returned, which stay
+ /// taken until the job ends).
+ /// </summary>
public void Release(IEnumerable<string> except)
{
var kept = except.ToHashSet(StringComparer.OrdinalIgnoreCase);
@@ -855,26 +933,31 @@ public sealed class JobQueue : IAsyncDisposable
foreach (var (path, _) in _allocated)
{
if (!kept.Contains(path))
- queue._reservedOutputs.Remove(path);
+ queue.Unreserve(path);
}
+ _allocated.Clear();
}
}
/// <summary>
- /// Undoes a conversion that failed or was cancelled: deletes what it started writing where nothing was before
- /// (never a file it was to overwrite), so no half-written file is left where the user expects a finished one,
- /// and frees every name, so trying again gets the same one instead of "name (2)".
+ /// Undoes a conversion that failed or was cancelled: deletes what it started writing at paths that were its
+ /// alone (never a file it was to overwrite, nor one another conversion had too), so no half-written file is left
+ /// where the user expects a finished one; frees every name, so trying again gets the same one instead of
+ /// "name (2)"; and removes a folder it made for them if that left it empty.
/// </summary>
public void Discard()
{
lock (queue._reservedOutputs)
{
- foreach (var (path, isNew) in _allocated)
+ foreach (var (path, own) in _allocated)
{
- if (isNew)
+ if (own && queue._reservedOutputs.TryGetValue(path, out var reservation) && !reservation.Shared)
TryDeleteOutput(path);
- queue._reservedOutputs.Remove(path);
+ queue.Unreserve(path);
}
+ foreach (var folder in _allocated.Select(a => Path.GetDirectoryName(Path.GetFullPath(a.Path))!)
+ .Distinct(Platform.FileSystemPaths.Comparer).ToList())
+ queue.RemoveIfEmpty(folder);
_allocated.Clear();
}
}
diff --git a/tests/Filee.Core.Tests/JobQueueTests.cs b/tests/Filee.Core.Tests/JobQueueTests.cs
index 6effd9af..dbddcb23 100644
--- a/tests/Filee.Core.Tests/JobQueueTests.cs
+++ b/tests/Filee.Core.Tests/JobQueueTests.cs
@@ -538,22 +538,28 @@ public class JobQueueTests
}
/// <summary>An engine that writes half a file where it was told to and then fails, or is cancelled.</summary>
- private sealed class HalfWriter(Action<ConversionStep>? whileWriting = null) : IConverter
+ private sealed class HalfWriter(Action<ConversionStep>? whileWriting = null, string from = "jpg") : IConverter
{
public string Id => "half";
public string DisplayName => "half";
- public IReadOnlyList<ConversionEdge> Edges { get; } = [new ConversionEdge("jpg", "png")];
+ public IReadOnlyList<ConversionEdge> Edges { get; } = [new ConversionEdge(from, "png")];
public int MaxParallelism => 0;
public bool Writes { get; init; } = true;
public bool Available { get; set; } = true;
+
+ /// <summary>Awaited after taking the name and before failing: lets a test run something else meanwhile.</summary>
+ public Func<Task>? Hold { get; init; }
+
public EngineStatus GetStatus() => Available ? EngineStatus.Available() : EngineStatus.Unavailable("engine.reason.not_installed");
- public Task<IReadOnlyList<string>> ConvertAsync(ConversionStep step, IProgress<double>? progress, CancellationToken cancellationToken)
+ public async Task<IReadOnlyList<string>> ConvertAsync(ConversionStep step, IProgress<double>? progress, CancellationToken cancellationToken)
{
var path = step.Output.Allocate(step.To)!;
if (Writes)
- File.WriteAllText(path, "half");
+ await File.WriteAllTextAsync(path, "half", cancellationToken);
whileWriting?.Invoke(step);
+ if (Hold is not null)
+ await Hold();
cancellationToken.ThrowIfCancellationRequested();
throw new InvalidOperationException("the engine crashed");
}
@@ -632,6 +638,45 @@ public class JobQueueTests
Assert.Equal("older", File.ReadAllText(older));
}
+ [Fact]
+ public async Task A_failed_conversion_never_deletes_the_file_another_one_was_given_the_same_name_for()
+ {
+ // "Overwrite" hands photo.gif and photo.jpg the same photo.png. The GIF's engine took it first and fails after
+ // the JPG's has finished it: that file stays.
+ using var dir = new TempDir();
+ var took = new TaskCompletionSource();
+ var fail = new TaskCompletionSource();
+ var half = new HalfWriter(_ => took.TrySetResult(), from: "gif") { Writes = false, Hold = () => fail.Task };
+ var good = new FakeConverter("magick", new ConversionEdge("jpg", "png"));
+ await using var queue = new JobQueue(new ConverterCatalog([half, good]));
+ var preset = new Preset { TargetFormat = "png", Output = { Conflict = ConflictPolicy.Overwrite } };
+
+ var failing = Run(queue, [dir.File("photo.gif")], preset);
+ await took.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken);
+ var finished = await Run(queue, [dir.File("photo.jpg")], preset);
+ fail.SetResult();
+
+ Assert.Equal(JobState.Failed, (await failing).State);
+ Assert.Equal([Path.Combine(dir.Path, "photo.png")], finished.Outputs);
+ Assert.Contains("by magick", File.ReadAllText(Path.Combine(dir.Path, "photo.png")));
+ }
+
+ [Fact]
+ public async Task A_failed_conversion_removes_the_subfolder_it_made_but_not_one_that_was_there()
+ {
+ using var dir = new TempDir();
+ await using var queue = new JobQueue(new ConverterCatalog([new HalfWriter()]));
+ var preset = new Preset { TargetFormat = "png", Output = { Location = OutputLocation.Subfolder, SubfolderName = "{name}" } };
+
+ Assert.Equal(JobState.Failed, (await Run(queue, [dir.File("photo.jpg")], preset)).State);
+ Assert.False(Directory.Exists(Path.Combine(dir.Path, "photo")));
+
+ var mine = Directory.CreateDirectory(Path.Combine(dir.Path, "scan")).FullName;
+ Assert.Equal(JobState.Failed, (await Run(queue, [dir.File("scan.jpg")], preset)).State);
+ Assert.True(Directory.Exists(mine));
+ Assert.Empty(Directory.EnumerateFileSystemEntries(mine));
+ }
+
[Fact]
public async Task Subfolder_only_for_several_files_keeps_a_single_output_next_to_its_source()
{