Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

fix(pdf): route text rhwp cannot measure around it when another route exists

Anatoly Kopteve6e8e0b4a547
diff --git a/docs/ENGINES.md b/docs/ENGINES.md
index 3ba038b0..7211ab53 100644
--- a/docs/ENGINES.md
+++ b/docs/ENGINES.md
@@ -49,8 +49,13 @@ RTF character references) for the Hebrew, Arabic, Syriac, Thaana, N'Ko, Indic, S
Khmer, Mongolian and other South / Southeast Asian blocks. For such a file `JobQueue` plans again without unshaped
edges: DOCX, TXT, Markdown, HTML, XLSX, PPTX and HWPX → PDF (and → images through PDF) go through LibreOffice even
when Settings → *Engines* doesn't choose it for documents. Without LibreOffice the file fails with "This script needs
-LibreOffice to convert correctly" and the script's name instead of becoming a broken PDF. Latin, Greek, Cyrillic, CJK
-and Korean documents keep the built-in route; a binary HWP is not scanned and stays with rhwp.
+LibreOffice to convert correctly" and the script's name instead of becoming a broken PDF. The same scan also finds
+text rhwp has no widths for: its width tables cover only ASCII and Latin-1, general punctuation, math, the CJK forms
+and Hangul (rhwp#8143), so a letter of any other script — Polish, Turkish, Vietnamese, Greek, Cyrillic, ... — or a
+combining mark is drawn but laid out at a fixed half-em. Such a document takes a route without unshaped edges when
+one exists and is never refused when none does: the overlapping text stays readable. Documents in French, German or
+Spanish (Latin-1 letters only), Chinese, Japanese and Korean keep the built-in route; a binary HWP is not scanned
+and stays with rhwp.
All built-in document readers are registered in one place, `Hwp/Hwpx/DocumentReaders.cs` (format id → reader, and
which formats need Pandoc). The HWPX writer and the DOCX writer take their input formats from it: adding a reader
diff --git a/src/Filee.Core/Conversion/IConverter.cs b/src/Filee.Core/Conversion/IConverter.cs
index 3a84b9d8..eb278a12 100644
--- a/src/Filee.Core/Conversion/IConverter.cs
+++ b/src/Filee.Core/Conversion/IConverter.cs
@@ -36,7 +36,9 @@ namespace Filee.Core.Conversion;
/// The engine draws text one character after the other, left to right, without complex-script shaping (no joined
/// Arabic letters, no right-to-left order, no reordered vowel signs, conjuncts or placed marks). A source whose text
/// needs shaping is never routed through the edge: <see cref="JobQueue"/> asks the engine
-/// (<see cref="ITextShapingCheck"/>) and plans again without it.
+/// (<see cref="ITextShapingCheck"/>) and plans again without it. Text the engine draws but has no widths for
+/// (letters outside ASCII and Latin-1 other than full-width CJK — Polish, Turkish, Vietnamese, Greek, Cyrillic —
+/// rhwp#8143) takes a route without the edge only when one exists.
/// </param>
/// <param name="MotionPhoto">
/// The edge takes the video out of a Live / Motion Photo (<see cref="Presets.MediaOptions.ExtractMotionPhoto"/>). It is
@@ -167,18 +169,35 @@ public interface IImageTransformer
bool Transforms(string from, string to);
}
+/// <summary>
+/// The scripts of a file's text an engine would draw wrong, from one scan.
+/// </summary>
+/// <param name="NeedsShaping">
+/// The name of the first script that needs complex-script shaping (e.g. "Gujarati"), or null. Takes precedence:
+/// <see cref="JobQueue"/> refuses a file with it (or routes it around the engine), never merely reroutes it for
+/// <paramref name="WithoutWidths"/>.
+/// </param>
+/// <param name="WithoutWidths">
+/// The name of the first script the engine draws but has no widths for (e.g. "Latin Extended" or "Cyrillic"), or
+/// null. Weaker than <paramref name="NeedsShaping"/>: such a file is not refused for it, <see cref="JobQueue"/>
+/// only takes a route without the unshaped edge when one exists.
+/// </param>
+public sealed record TextScripts(string? NeedsShaping, string? WithoutWidths);
+
/// <summary>
/// An engine with <see cref="ConversionEdge.Unshaped"/> edges: tells whether a source file has text that needs
-/// complex-script shaping (Arabic, Hebrew, the Indic scripts, Thai, ...), which those edges would draw broken.
+/// complex-script shaping (Arabic, Hebrew, the Indic scripts, Thai, ...), which those edges would draw broken,
+/// or text the engine has no widths for (letters outside ASCII and Latin-1 other than full-width CJK — Polish,
+/// Turkish, Vietnamese, Greek, Cyrillic, ...), which they lay out wrong.
/// </summary>
public interface ITextShapingCheck
{
/// <summary>
- /// The name of the first script in the text of <paramref name="sourcePath"/> that needs shaping (e.g. "Gujarati"),
- /// or null. Quick: it scans the file's text, it doesn't convert it.
+ /// Scans the text of <paramref name="sourcePath"/> once and answers both questions. Quick: it scans the file's
+ /// text, it doesn't convert it — and it is asked again when the file is converted again, never cached.
/// </summary>
/// <param name="sourceFormat">Format id of the file.</param>
- string? ScriptNeedingShaping(string sourcePath, string sourceFormat);
+ TextScripts ScanScripts(string sourcePath, string sourceFormat);
}
/// <summary>Merges several PDF files into one. Implemented by the PDF engine.</summary>
diff --git a/src/Filee.Core/Conversion/JobQueue.cs b/src/Filee.Core/Conversion/JobQueue.cs
index 9cf06228..7a162beb 100644
--- a/src/Filee.Core/Conversion/JobQueue.cs
+++ b/src/Filee.Core/Conversion/JobQueue.cs
@@ -502,9 +502,12 @@ public sealed class JobQueue : IAsyncDisposable
return Fail(file, "error.no_route", $"{source.Id} → {target}");
// A step that draws text without shaping (rhwp) would break Arabic, Hebrew, Indic or Thai text: such a file
- // takes a route without it (LibreOffice), or is refused with the reason rather than written broken.
- if (route.Steps.FirstOrDefault(s => s.Unshaped)?.Converter is ITextShapingCheck check
- && ScriptNeedingShaping(check, file.SourcePath, source.Id) is { } script)
+ // takes a route without it (LibreOffice), or is refused with the reason rather than written broken. One
+ // scan of the file's text answers this and the widths question below.
+ var scripts = route.Steps.FirstOrDefault(s => s.Unshaped)?.Converter is ITextShapingCheck check
+ ? ScanScripts(check, file.SourcePath, source.Id)
+ : null;
+ if (scripts?.NeedsShaping is { } script)
{
_log.LogInformation("{File} has {Script} text, which {Engine} can't draw: planning a route without it",
file.SourcePath, script, route.Steps.First(s => s.Unshaped).Converter.DisplayName);
@@ -512,6 +515,16 @@ public sealed class JobQueue : IAsyncDisposable
if (route is null)
return Fail(file, "error.needs_shaping", script);
}
+ // Letters the engine has no widths for — anything outside ASCII and Latin-1 but the full-width CJK it
+ // estimates at an em (Polish, Turkish, Vietnamese, Greek, Cyrillic, ... — rhwp#8143) — overlap at a fixed
+ // half-em: still readable, so the file is never refused, it only takes a route without the unshaped step
+ // when there is one.
+ else if (scripts?.WithoutWidths is { } unmeasured)
+ {
+ _log.LogInformation("{File} has {Script} text, which {Engine} can't measure: planning a route without it",
+ file.SourcePath, unmeasured, route.Steps.First(s => s.Unshaped).Converter.DisplayName);
+ route = planner.Plan(source.Id, target, frames, shaping: true) ?? route;
+ }
// Rotate / flip: only an engine that turns images does it. Without one the file would come out unturned
// (a PDF or a video re-encoded "as rotated"); with two in the route (EPS → PNG → JPG) it would turn twice.
@@ -683,12 +696,12 @@ public sealed class JobQueue : IAsyncDisposable
return [zipPath];
}
- /// <summary>The engine's check; a check that fails (an unreadable file) finds nothing, and the conversion reports the file.</summary>
- private string? ScriptNeedingShaping(ITextShapingCheck check, string path, string format)
+ /// <summary>The engine's scan; one that fails (an unreadable file) finds nothing, and the conversion reports the file.</summary>
+ private TextScripts? ScanScripts(ITextShapingCheck check, string path, string format)
{
try
{
- return check.ScriptNeedingShaping(path, format);
+ return check.ScanScripts(path, format);
}
catch (Exception ex)
{
diff --git a/src/Filee.Core/Conversion/RoutePlanner.cs b/src/Filee.Core/Conversion/RoutePlanner.cs
index b19047eb..52de3ae1 100644
--- a/src/Filee.Core/Conversion/RoutePlanner.cs
+++ b/src/Filee.Core/Conversion/RoutePlanner.cs
@@ -62,7 +62,9 @@ public sealed class RoutePlanner
/// Without it those edges are never taken.
/// </param>
/// <param name="shaping">
- /// The source has text that needs complex-script shaping: <see cref="ConversionEdge.Unshaped"/> edges are left out.
+ /// The source has text that needs complex-script shaping — or letters the unshaped engine has no widths for
+ /// (outside ASCII and Latin-1 other than full-width CJK): <see cref="ConversionEdge.Unshaped"/> edges are
+ /// left out.
/// </param>
/// <param name="motionPhoto">
/// Take the video of a Live / Motion Photo (<see cref="Presets.Preset.ExtractsMotionOf"/>): the route is one
diff --git a/src/Filee.Engines/Hwp/RhwpConverter.cs b/src/Filee.Engines/Hwp/RhwpConverter.cs
index 27e39544..81485009 100644
--- a/src/Filee.Engines/Hwp/RhwpConverter.cs
+++ b/src/Filee.Engines/Hwp/RhwpConverter.cs
@@ -4,7 +4,10 @@
// rhwp draws text one character after the other, left to right, without OpenType shaping: Arabic letters don't join
// and run the wrong way, Indic vowel signs and conjuncts come out as separate glyphs with dotted circles (issues #185,
// #202). Its PDF edges are marked Unshaped, so a document with such text (found by ComplexScripts) goes to LibreOffice,
-// or is refused with that reason when LibreOffice is missing.
+// or is refused with that reason when LibreOffice is missing. rhwp's width tables cover only ASCII and Latin-1,
+// general punctuation, math, the CJK forms and Hangul, so a letter of any other script — Polish, Turkish, Vietnamese,
+// Greek, Cyrillic, ... — overlaps at a fixed half-em (rhwp#8143): such a document takes a route without rhwp when
+// one exists, and is never refused for it.
using System.IO.Compression;
using Filee.Core.Conversion;
@@ -44,10 +47,16 @@ public sealed class RhwpConverter : IConverter, ITextShapingCheck
];
/// <summary>
- /// The first script in the source's text that rhwp can't draw. Every format that reaches rhwp through the HWPX
- /// writer is scanned (DOCX, TXT, Markdown, HTML, PPTX, XLSX, EPUB, HWPX, ...); a binary HWP is not.
+ /// The scripts in the source's text that rhwp can't draw: one that needs shaping, and one it draws but has no
+ /// widths for — letters outside ASCII and Latin-1 other than the full-width CJK it estimates at an em
+ /// (rhwp#8143). Both answers come from one pass. Every format that reaches rhwp through the HWPX writer is
+ /// scanned (DOCX, TXT, Markdown, HTML, PPTX, XLSX, EPUB, HWPX, ...); a binary HWP is not.
/// </summary>
- public string? ScriptNeedingShaping(string sourcePath, string sourceFormat) => ComplexScripts.FindInFile(sourcePath, sourceFormat);
+ public TextScripts ScanScripts(string sourcePath, string sourceFormat)
+ {
+ var (shaping, unmeasured) = ComplexScripts.FindBothInFile(sourcePath, sourceFormat);
+ return new TextScripts(shaping, unmeasured);
+ }
public EngineStatus GetStatus()
{
diff --git a/src/Filee.Engines/Infrastructure/ComplexScripts.cs b/src/Filee.Engines/Infrastructure/ComplexScripts.cs
index 56d17b49..9265fca6 100644
--- a/src/Filee.Engines/Infrastructure/ComplexScripts.cs
+++ b/src/Filee.Engines/Infrastructure/ComplexScripts.cs
@@ -7,7 +7,13 @@
// FindInFile scans a document's text cheaply, without reading it into the document model: the content parts of an
// Office Open XML / OWPML / OpenDocument / EPUB package as they are (markup is ASCII, so any match is text), a text
// file decoded like the plain text reader does (character references of HTML and RTF included).
+//
+// The same scan also finds text rhwp has no widths for: its width tables cover only ASCII and Latin-1, general
+// punctuation, math, the CJK forms and Hangul (rhwp#8143), so a letter of any other script — Polish, Turkish,
+// Vietnamese, Greek, Cyrillic, ... — or a combining mark is laid out at a fixed half-em. Such a document takes
+// a route without unshaped edges when one exists and keeps rhwp otherwise (it is never refused for this).
+using System.Globalization;
using System.IO.Compression;
using System.Text;
using System.Text.RegularExpressions;
@@ -74,6 +80,72 @@ internal static partial class ComplexScripts
(0x11000, 0x11FFF, "Brahmic"),
];
+ /// <summary>
+ /// The code point ranges rhwp's width tables are generated for (src/tools/font_metric_gen.rs, rhwp#8143):
+ /// ASCII, Latin-1, general punctuation, math, CJK symbols, Hangul jamo and syllables, and the fullwidth
+ /// forms. A rhwp release that measures more is a one-line edit here.
+ /// </summary>
+ private static readonly (int Start, int End)[] MeasuredRanges =
+ [
+ (0x0020, 0x007E),
+ (0x00A0, 0x00FF),
+ (0x2000, 0x206F),
+ (0x2200, 0x22FF),
+ (0x3000, 0x303F),
+ (0x3130, 0x318F),
+ (0xAC00, 0xD7A3),
+ (0xFF00, 0xFF5E),
+ ];
+
+ /// <summary>Blocks rhwp does not measure but lays out at a full em, which is right for them (rhwp#8143).</summary>
+ private static readonly (int Start, int End)[] FullWidthCjk =
+ [
+ (0x1100, 0x11FF),
+ (0x2E80, 0x2FDF),
+ (0x3000, 0x303F),
+ (0x3040, 0x30FF),
+ (0x3100, 0x312F),
+ (0x3130, 0x318F),
+ (0x31A0, 0x31FF),
+ (0x3200, 0x33FF),
+ (0x3400, 0x4DBF),
+ (0x4E00, 0x9FFF),
+ (0xA960, 0xA97F),
+ (0xAC00, 0xD7FF),
+ (0xF900, 0xFAFF),
+ (0xFF00, 0xFFEF),
+ (0x20000, 0x3FFFF),
+ ];
+
+ /// <summary>
+ /// Variation selectors: zero-width, default-ignorable marks (Mn) that only pick the glyph variant of the
+ /// letter they follow — an emoji's text or picture form, a Mongolian letter's shape, an ideograph's
+ /// registered variant. They carry no width of their own, so they must not reroute a measured document.
+ /// </summary>
+ private static readonly (int Start, int End)[] VariationSelectors =
+ [
+ (0x180B, 0x180D),
+ (0xFE00, 0xFE0F),
+ (0xE0100, 0xE01EF),
+ ];
+
+ /// <summary>Coarse names for the log line, by first code point like <see cref="Blocks"/>; else "U+XXXX".</summary>
+ private static readonly (int Start, int End, string Name)[] UnmeasuredNames =
+ [
+ (0x0100, 0x024F, "Latin Extended"),
+ (0x0250, 0x02AF, "IPA"),
+ (0x0300, 0x036F, "combining marks"),
+ (0x0370, 0x03FF, "Greek"),
+ (0x0400, 0x052F, "Cyrillic"),
+ (0x0530, 0x058F, "Armenian"),
+ (0x10A0, 0x10FF, "Georgian"),
+ (0x1C80, 0x1C8F, "Cyrillic"),
+ (0x1E00, 0x1EFF, "Latin Extended"),
+ (0x1F00, 0x1FFF, "Greek"),
+ (0x2DE0, 0x2DFF, "Cyrillic"),
+ (0xA640, 0xA69F, "Cyrillic"),
+ ];
+
/// <summary>The script of <paramref name="codePoint"/> when it needs shaping ("Arabic", "Gujarati"), else null.</summary>
public static string? ScriptOf(int codePoint)
{
@@ -90,6 +162,48 @@ internal static partial class ComplexScripts
return null;
}
+ /// <summary>
+ /// The script of <paramref name="codePoint"/> when rhwp draws it but has no width for it ("Cyrillic"), else
+ /// null: a letter or combining mark outside <see cref="MeasuredRanges"/> — decomposed e + ´ is drawn at the
+ /// same half-em — that is no full-width CJK rhwp estimates at an em (<see cref="FullWidthCjk"/>) and no
+ /// shaping script (that is the other scan's job). The name only reaches the log;
+ /// <see cref="UnmeasuredNames"/> is intentionally coarse.
+ /// </summary>
+ private static string? UnmeasuredScriptOf(int codePoint)
+ {
+ if (codePoint is < 0 or > 0x10FFFF // a reference can name a value outside Unicode
+ || Inside(VariationSelectors, codePoint)
+ || CharUnicodeInfo.GetUnicodeCategory(codePoint)
+ is not (UnicodeCategory.UppercaseLetter or UnicodeCategory.LowercaseLetter
+ or UnicodeCategory.TitlecaseLetter or UnicodeCategory.ModifierLetter or UnicodeCategory.OtherLetter
+ or UnicodeCategory.NonSpacingMark or UnicodeCategory.SpacingCombiningMark)
+ || Inside(MeasuredRanges, codePoint)
+ || Inside(FullWidthCjk, codePoint)
+ || ScriptOf(codePoint) is not null)
+ return null;
+ foreach (var (start, end, name) in UnmeasuredNames)
+ {
+ if (codePoint < start)
+ break;
+ if (codePoint <= end)
+ return name;
+ }
+ return $"U+{codePoint:X4}";
+ }
+
+ /// <summary>True when <paramref name="codePoint"/> is inside one of the sorted <paramref name="ranges"/>.</summary>
+ private static bool Inside((int Start, int End)[] ranges, int codePoint)
+ {
+ foreach (var (start, end) in ranges)
+ {
+ if (codePoint < start)
+ return false;
+ if (codePoint <= end)
+ return true;
+ }
+ return false;
+ }
+
public static bool IsComplex(int codePoint) => ScriptOf(codePoint) is not null;
/// <summary>A letter of a right-to-left script (Hebrew, Arabic, Syriac, Thaana, N'Ko, ...).</summary>
@@ -130,17 +244,23 @@ internal static partial class ComplexScripts
}
/// <summary>The first script in <paramref name="text"/> that needs shaping, or null.</summary>
- public static string? Find(ReadOnlySpan<char> text)
+ public static string? Find(ReadOnlySpan<char> text) => Find(text, ScriptOf, Blocks[0].Start);
+
+ /// <summary>
+ /// The first answer of <paramref name="classifier"/> in <paramref name="text"/>; code points below
+ /// <paramref name="skipBelow"/> (the classifier's first block) are not asked about.
+ /// </summary>
+ private static string? Find(ReadOnlySpan<char> text, Func<int, string?> classifier, int skipBelow)
{
for (var i = 0; i < text.Length; i++)
{
var c = text[i];
- if (c < 0x0590)
+ if (c < skipBelow)
continue;
var codePoint = char.IsHighSurrogate(c) && i + 1 < text.Length && char.IsLowSurrogate(text[i + 1])
? char.ConvertToUtf32(c, text[++i])
: c;
- if (ScriptOf(codePoint) is { } script)
+ if (classifier(codePoint) is { } script)
return script;
}
return null;
@@ -154,21 +274,50 @@ internal static partial class ComplexScripts
/// here: the conversion then reports what is wrong with them).
/// </summary>
/// <param name="format">Format id of the file (docx, txt, hwpx, ...).</param>
- public static string? FindInFile(string path, string format)
+ public static string? FindInFile(string path, string format) => FindInFile(path, format, ScriptOf, Blocks[0].Start);
+
+ /// <summary>
+ /// The first script in the text of a document that rhwp draws but cannot measure (rhwp#8143), or null.
+ /// </summary>
+ /// <param name="format">Format id of the file (docx, txt, hwpx, ...).</param>
+ public static string? FindUnmeasuredInFile(string path, string format) => FindBothInFile(path, format).Unmeasured;
+
+ /// <summary>
+ /// Both answers of the scan in one pass: the first script that needs shaping stops the scan and wins the
+ /// file; the first script rhwp cannot measure is remembered while the scan runs on.
+ /// </summary>
+ /// <param name="format">Format id of the file (docx, txt, hwpx, ...).</param>
+ public static (string? Shaping, string? Unmeasured) FindBothInFile(string path, string format)
+ {
+ string? unmeasured = null;
+ // No letter below 0x0100 is unmeasured: ASCII and Latin-1 are, controls are no letters.
+ var shaping = FindInFile(path, format, Both, skipBelow: 0x0100);
+ return (shaping, unmeasured);
+
+ string? Both(int codePoint)
+ {
+ if (ScriptOf(codePoint) is { } script)
+ return script;
+ unmeasured ??= UnmeasuredScriptOf(codePoint);
+ return null;
+ }
+ }
+
+ private static string? FindInFile(string path, string format, Func<int, string?> classifier, int skipBelow)
{
try
{
if (IsZip(path))
- return FindInPackage(path, format);
+ return FindInPackage(path, format, classifier, skipBelow);
// Web archives and mail are MIME: their HTML (or text) part is what reaches the PDF.
if (format is "mhtml" or "eml")
{
var message = MimeKit.MimeMessage.Load(path);
- return FindInString(message.HtmlBody ?? message.TextBody ?? "");
+ return FindInString(message.HtmlBody ?? message.TextBody ?? "", classifier, skipBelow);
}
return format is "txt" or "md" or "html" or "csv" or "tsv" or "fb2" or "rtf" or "rst" or "tex" or "org" or "adoc"
or "typ" or "mediawiki" or "dbk" or "textile" or "man" or "opml"
- ? FindInText(path)
+ ? FindInText(path, classifier, skipBelow)
: null; // binary formats (HWP, DOC, MOBI, ...): left to the engines
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException or FormatException)
@@ -187,7 +336,7 @@ internal static partial class ComplexScripts
/// <summary>Text files are read whole (as the readers do), up to this size; a larger one is scanned in its first part.</summary>
private const int MaxTextBytes = 64 * 1024 * 1024;
- private static string? FindInText(string path)
+ private static string? FindInText(string path, Func<int, string?> classifier, int skipBelow)
{
byte[] bytes;
using (var stream = File.OpenRead(path))
@@ -195,12 +344,12 @@ internal static partial class ComplexScripts
bytes = new byte[(int)Math.Min(stream.Length, MaxTextBytes)];
stream.ReadExactly(bytes);
}
- return FindInString(DocumentReaders.DecodeText(bytes));
+ return FindInString(DocumentReaders.DecodeText(bytes), classifier, skipBelow);
}
- private static string? FindInString(string text)
+ private static string? FindInString(string text, Func<int, string?> classifier, int skipBelow)
{
- if (Find(text) is { } script)
+ if (Find(text, classifier, skipBelow) is { } script)
return script;
// &#2693; / &#x0A85; in HTML and XML, ⚓? in RTF.
foreach (Match match in CharacterReference().Matches(text))
@@ -210,7 +359,9 @@ internal static partial class ComplexScripts
: int.Parse(match.Groups["dec"].ValueSpan, System.Globalization.CultureInfo.InvariantCulture);
if (value < 0)
value += 0x10000; // RTF writes code points above 32767 as negative numbers
- if (ScriptOf(value) is { } referenced)
+ if (value is < 0 or > 0x10FFFF)
+ continue; // a reference can name a value outside Unicode (&#1114112;, \u-99999): it is no script
+ if (classifier(value) is { } referenced)
return referenced;
}
return null;
@@ -242,7 +393,7 @@ internal static partial class ComplexScripts
};
}
- private static string? FindInPackage(string path, string format)
+ private static string? FindInPackage(string path, string format, Func<int, string?> classifier, int skipBelow)
{
using var zip = ZipFile.OpenRead(path);
var buffer = new char[64 * 1024];
@@ -255,7 +406,7 @@ internal static partial class ComplexScripts
while ((read = reader.Read(buffer, 0, buffer.Length)) > 0)
{
// A surrogate pair cut at the end of the buffer only hides a supplementary-plane letter there.
- if (Find(buffer.AsSpan(0, read)) is { } script)
+ if (Find(buffer.AsSpan(0, read), classifier, skipBelow) is { } script)
return script;
}
}
diff --git a/tests/Filee.Core.Tests/JobQueueTests.cs b/tests/Filee.Core.Tests/JobQueueTests.cs
index bb50d70c..8d0e68f3 100644
--- a/tests/Filee.Core.Tests/JobQueueTests.cs
+++ b/tests/Filee.Core.Tests/JobQueueTests.cs
@@ -110,16 +110,19 @@ public class JobQueueTests
public string DisplayName => id;
public IReadOnlyList<ConversionEdge> Edges => edges;
public int MaxParallelism => 0;
+ /// <summary>Files <see cref="ScanScripts"/> was called for.</summary>
public List<string> Checked { get; } = [];
+ /// <summary>The <see cref="TextScripts.WithoutWidths"/> answer for a file (default: everything is measured).</summary>
+ public Func<string, string?> WithoutWidths { get; set; } = _ => null;
private int _converted;
public int Converted => _converted;
public EngineStatus GetStatus() => EngineStatus.Available();
- public string? ScriptNeedingShaping(string sourcePath, string sourceFormat)
+ public TextScripts ScanScripts(string sourcePath, string sourceFormat)
{
lock (Checked)
Checked.Add(sourcePath);
- return script(sourcePath);
+ return new TextScripts(script(sourcePath), WithoutWidths(sourcePath));
}
public Task<IReadOnlyList<string>> ConvertAsync(ConversionStep step, IProgress<double>? progress, CancellationToken cancellationToken)
@@ -129,6 +132,64 @@ public class JobQueueTests
}
}
+ [Fact]
+ public async Task Text_the_engine_cannot_measure_takes_another_route_but_is_never_refused()
+ {
+ // rhwp#8143: rhwp's width tables don't cover Cyrillic, so its letters overlap — ugly but readable. Such a
+ // file takes a route around the unshaped engine when there is one, and still converts when there isn't.
+ using var dir = new TempDir();
+ var russian = dir.File("russian.docx");
+ var english = dir.File("english.docx");
+ var writer = new FakeConverter("hwpx-writer", new ConversionEdge("docx", "hwpx"));
+ var rhwp = new ShapingCheck("rhwp", _ => null, new ConversionEdge("hwpx", "pdf", Unshaped: true))
+ {
+ WithoutWidths = path => Path.GetFileName(path).StartsWith("russian", StringComparison.Ordinal) ? "Cyrillic" : null,
+ };
+ var office = new FakeConverter("libreoffice", new ConversionEdge("docx", "pdf", 25));
+ var preset = new Preset { TargetFormat = "pdf" };
+
+ await using (var queue = new JobQueue(new ConverterCatalog([writer, rhwp, office])))
+ {
+ var job = await Run(queue, [russian, english], preset);
+ Assert.Equal(JobState.Completed, job.State);
+ Assert.Equal(1, rhwp.Converted); // English only
+ Assert.Equal("russian.docx→pdf by libreoffice", File.ReadAllText(Path.Combine(dir.Path, "russian.pdf")));
+ Assert.Equal(["english.docx", "russian.docx"], rhwp.Checked.Order().Select(Path.GetFileName));
+ }
+
+ office.Available = false;
+ await using (var queue = new JobQueue(new ConverterCatalog([writer, rhwp, office])))
+ {
+ var job = await Run(queue, [dir.File("russian-2.docx")], preset);
+ var file = Assert.Single(job.Files);
+ Assert.Equal(FileState.Done, file.State);
+ Assert.EndsWith("→pdf by rhwp", File.ReadAllText(Path.Combine(dir.Path, "russian-2.pdf")));
+ }
+ }
+
+ [Fact]
+ public async Task Text_with_shaping_and_unmeasured_scripts_is_refused_for_shaping()
+ {
+ // A scan finding both (Arabic + Cyrillic): shaping takes precedence, the file fails with its reason and is
+ // never converted through the widths escape (a route kept only because the overlap stays readable).
+ using var dir = new TempDir();
+ var arabic = dir.File("arabic.docx");
+ var writer = new FakeConverter("hwpx-writer", new ConversionEdge("docx", "hwpx"));
+ var rhwp = new ShapingCheck("rhwp", _ => "Arabic", new ConversionEdge("hwpx", "pdf", Unshaped: true))
+ {
+ WithoutWidths = _ => "Cyrillic",
+ };
+ await using var queue = new JobQueue(new ConverterCatalog([writer, rhwp]));
+
+ var job = await Run(queue, [arabic], new Preset { TargetFormat = "pdf" });
+
+ var file = Assert.Single(job.Files);
+ Assert.Equal(FileState.Failed, file.State);
+ Assert.Equal("error.needs_shaping", file.ErrorKey);
+ Assert.Equal("Arabic", file.ErrorDetail);
+ Assert.Equal(0, rhwp.Converted);
+ }
+
[Fact]
public async Task Rotate_needs_an_engine_that_turns_images_and_turns_once_on_a_longer_route()
{
diff --git a/tests/Filee.Engines.Tests/ComplexScriptTests.cs b/tests/Filee.Engines.Tests/ComplexScriptTests.cs
index 3bf58966..49d4389c 100644
--- a/tests/Filee.Engines.Tests/ComplexScriptTests.cs
+++ b/tests/Filee.Engines.Tests/ComplexScriptTests.cs
@@ -10,6 +10,7 @@ using System.Text;
using System.Xml.Linq;
using Filee.Core.Conversion;
using Filee.Core.Presets;
+using Filee.Engines.Hwp;
using Filee.Engines.Infrastructure;
using Filee.Engines.Pdf;
using static Filee.Engines.Tests.DocxAssert;
@@ -76,6 +77,97 @@ public class ComplexScriptTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.Null(ComplexScripts.FindInFile(Ofd("chinese.ofd", "电子发票(普通发票)"), "ofd"));
}
+ [Fact]
+ public void Scripts_without_widths_are_found_in_documents()
+ {
+ // rhwp#8143: rhwp's width tables cover only ASCII, Latin-1, general punctuation, math, the CJK forms and
+ // Hangul, so a letter of any other script — Polish, Turkish, Vietnamese, Greek, Cyrillic, ... — is drawn
+ // but laid out at a fixed half-em.
+ var folder = fx.NewFolder();
+ string Docx(string name, string text)
+ {
+ var builder = new DocxBuilder();
+ builder.Paragraph(DocxBuilder.P(DocxBuilder.R(text)));
+ return builder.Save(Path.Combine(folder, name));
+ }
+ string Text(string name, string text)
+ {
+ var path = Path.Combine(folder, name);
+ File.WriteAllText(path, text, new UTF8Encoding(false));
+ return path;
+ }
+
+ Assert.Equal("Latin Extended", ComplexScripts.FindUnmeasuredInFile(Docx("polish.docx", "Zażółć gęślą jaźń"), "docx"));
+ Assert.Equal("Latin Extended", ComplexScripts.FindUnmeasuredInFile(Text("vietnamese.md", "Tiếng Việt"), "md"));
+ Assert.Equal("Greek", ComplexScripts.FindUnmeasuredInFile(Docx("greek.docx", "Αναφορά Report"), "docx"));
+ Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Docx("russian.docx", "Отчёт за месяц"), "docx"));
+ Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("russian.txt", "Заголовок\nтекст"), "txt"));
+ Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("russian.md", "# Отчёт\n\ntext"), "md"));
+ Assert.Equal("Armenian", ComplexScripts.FindUnmeasuredInFile(Text("armenian.txt", "Հայերեն տեքստ"), "txt"));
+ Assert.Equal("Georgian", ComplexScripts.FindUnmeasuredInFile(Text("georgian.txt", "ქართული ტექსტი"), "txt"));
+ // A precomposed é is Latin-1 and measured; e + ´ decomposed is not.
+ Assert.Equal("combining marks", ComplexScripts.FindUnmeasuredInFile(Text("decomposed.md", "cafe\u0301"), "md"));
+
+ // Latin-1 letters are measured, Hangul, CJK and the full-width forms are estimated right at a full em,
+ // and text that needs shaping is the other scan's job.
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("latin1.docx", "Ça été über Straße niño"), "docx"));
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("korean.docx", "한국어 문서"), "docx"));
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("japanese.docx", "日本語のテキスト"), "docx"));
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("chinese.docx", "中文文档"), "docx"));
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("fullwidth.docx", "ABC"), "docx"));
+ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("arabic.docx", "مرحبا بالعالم"), "docx"));
+ Assert.Null(ComplexScripts.FindInFile(Docx("russian-2.docx", "Отчёт"), "docx"));
+ }
+
+ [Fact]
+ public void A_character_reference_outside_unicode_is_skipped()
+ {
+ // &#1114112; (0x110000) and RTF \u-99999 (still negative past +0x10000) name no code point: a malformed
+ // reference before a document's letters must not lose both answers of the scan to an exception.
+ var folder = fx.NewFolder();
+ string Text(string name, string text)
+ {
+ var path = Path.Combine(folder, name);
+ File.WriteAllText(path, text, new UTF8Encoding(false));
+ return path;
+ }
+
+ var html = ComplexScripts.FindBothInFile(Text("bad-ref.html", "<p>&#1114112; &#x5E9;&#x5DC;</p>"), "html");
+ Assert.Equal("Hebrew", html.Shaping);
+
+ var rtf = ComplexScripts.FindBothInFile(Text("bad-ref.rtf", @"{\rtf1 \u-99999? ᒈ\u1513?\u1500?}"), "rtf");
+ Assert.Equal("Hebrew", rtf.Shaping);
+ }
+
+ [Fact]
+ public void Variation_selectors_are_not_unmeasured_text()
+ {
+ // U+FE0F picks an emoji's picture form (❤️, ✔️), U+E0100 an ideograph's registered variant (葛󠄀): the
+ // zero-width mark rides on a letter rhwp measures, so it must not reroute the document.
+ var folder = fx.NewFolder();
+ var builder = new DocxBuilder();
+ builder.Paragraph(DocxBuilder.P(DocxBuilder.R("Déjà vu ❤️ ✔️")));
+ var docx = builder.Save(Path.Combine(folder, "emoji.docx"));
+ var txt = Path.Combine(folder, "ivs.txt");
+ File.WriteAllText(txt, "葛\U000E0100の本文", new UTF8Encoding(false));
+
+ Assert.Null(ComplexScripts.FindBothInFile(docx, "docx").Unmeasured);
+ Assert.Null(ComplexScripts.FindBothInFile(txt, "txt").Unmeasured);
+ }
+
+ [Fact]
+ public void Both_answers_come_from_one_scan()
+ {
+ // Cyrillic before Arabic: the scan remembers the first unmeasured script while it looks on for shaping
+ // text, so the file keeps both answers.
+ var path = Path.Combine(fx.NewFolder(), "mixed.txt");
+ File.WriteAllText(path, "Отчёт مرحبا", new UTF8Encoding(false));
+
+ var (shaping, unmeasured) = ComplexScripts.FindBothInFile(path, "txt");
+ Assert.Equal("Arabic", shaping);
+ Assert.Equal("Cyrillic", unmeasured);
+ }
+
[Fact]
public async Task Scripts_are_found_in_hwpx_sections()
{
@@ -98,6 +190,23 @@ public class ComplexScriptTests(EngineFixture fx) : IClassFixture<EngineFixture>
Assert.IsAssignableFrom<ITextShapingCheck>(rhwp);
}
+ [Fact]
+ public void A_file_is_scanned_again_once_its_text_changed()
+ {
+ // A file converted twice in a session — a watch folder, a document saved again — is scanned again:
+ // an answer cached on the path would route its new text the old way.
+ var path = Path.Combine(fx.NewFolder(), "report.md");
+ File.WriteAllText(path, "A plain English report.", new UTF8Encoding(false));
+ var rhwp = new RhwpConverter();
+
+ var clean = rhwp.ScanScripts(path, "md");
+ Assert.Null(clean.NeedsShaping);
+ Assert.Null(clean.WithoutWidths);
+
+ File.WriteAllText(path, "مرحبا بالعالم", new UTF8Encoding(false));
+ Assert.Equal("Arabic", rhwp.ScanScripts(path, "md").NeedsShaping);
+ }
+
[Fact]
public async Task A_gujarati_document_goes_to_libreoffice_or_is_refused_with_the_reason()
{