Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.
C# audio-converteravaloniabatch-conversionconvertercsharpdocument-converterdocx-to-pdfdotnetdrag-and-dropfile-conversionfile-converterheic-converterhwphwpximage-converterlinuxmacospdf-convertervideo-converterwindows
Merge main; count unmeasured letters only inside words, keep HWPX with rhwp
KnifeLemon693842ab160b
diff --git a/docs/ENGINES.md b/docs/ENGINES.mdindex 7211ab53..23817d98 100644--- a/docs/ENGINES.md+++ b/docs/ENGINES.md@@ -53,9 +53,11 @@ LibreOffice to convert correctly" and the script's name instead of becoming a brtext rhwp has no widths for: its width tables cover only ASCII and Latin-1, general punctuation, math, the CJK formsand Hangul (rhwp#8143), so a letter of any other script — Polish, Turkish, Vietnamese, Greek, Cyrillic, ... — or acombining mark is drawn but laid out at a fixed half-em. Such a document takes a route without unshaped edges when-one exists and is never refused when none does: the overlapping text stays readable. Documents in French, German or-Spanish (Latin-1 letters only), Chinese, Japanese and Korean keep the built-in route; a binary HWP is not scanned-and stays with rhwp.+one exists and is never refused when none does: the overlapping text stays readable. Only letters inside words+count (an extended Latin letter next to any letter, a letter of another script next to one of its own script), so a+symbol or unit on its own (α, Ω, μm in a Korean report) keeps rhwp, and an HWPX stays with rhwp for this (only text+that needs shaping sends it elsewhere). Documents in French, German or Spanish (Latin-1 letters only), Chinese,+Japanese and Korean keep the built-in route; a binary HWP is not scanned and stays with rhwp.All built-in document readers are registered in one place, `Hwp/Hwpx/DocumentReaders.cs` (format id → reader, andwhich formats need Pandoc). The HWPX writer and the DOCX writer take their input formats from it: adding a readerdiff --git a/src/Filee.Engines/Hwp/RhwpConverter.cs b/src/Filee.Engines/Hwp/RhwpConverter.csindex 81485009..3c65e48b 100644--- a/src/Filee.Engines/Hwp/RhwpConverter.cs+++ b/src/Filee.Engines/Hwp/RhwpConverter.cs@@ -55,7 +55,8 @@ public sealed class RhwpConverter : IConverter, ITextShapingCheckpublic TextScripts ScanScripts(string sourcePath, string sourceFormat){var (shaping, unmeasured) = ComplexScripts.FindBothInFile(sourcePath, sourceFormat);- return new TextScripts(shaping, unmeasured);+ // An HWPX stays with rhwp, the most faithful renderer for it; only shaping sends it elsewhere.+ return new TextScripts(shaping, sourceFormat == "hwpx" ? null : unmeasured);}public EngineStatus GetStatus()diff --git a/src/Filee.Engines/Infrastructure/ComplexScripts.cs b/src/Filee.Engines/Infrastructure/ComplexScripts.csindex 9265fca6..a7fccaa7 100644--- a/src/Filee.Engines/Infrastructure/ComplexScripts.cs+++ b/src/Filee.Engines/Infrastructure/ComplexScripts.cs@@ -244,23 +244,36 @@ internal static partial class ComplexScripts}/// <summary>The first script in <paramref name="text"/> that needs shaping, or null.</summary>- public static string? Find(ReadOnlySpan<char> text) => Find(text, ScriptOf, Blocks[0].Start);+ public static string? Find(ReadOnlySpan<char> text) => Find(text, Shaping, Blocks[0].Start);++ /// <summary>+ /// Names the script of a code point, or null. <c>before</c> and <c>after</c> are the characters around it, or+ /// <see cref="Unknown"/> for a character reference.+ /// </summary>+ private delegate string? Classifier(int codePoint, int before, int after);++ /// <summary>Neighbour of a character reference: not known, so the reference counts as part of a word.</summary>+ private const int Unknown = -2;++ private static string? Shaping(int codePoint, int before, int after) => ScriptOf(codePoint);/// <summary>/// The first answer of <paramref name="classifier"/> in <paramref name="text"/>; code points below/// <paramref name="skipBelow"/> (the classifier's first block) are not asked about./// </summary>- private static string? Find(ReadOnlySpan<char> text, Func<int, string?> classifier, int skipBelow)+ private static string? Find(ReadOnlySpan<char> text, Classifier classifier, int skipBelow){for (var i = 0; i < text.Length; i++){var c = text[i];if (c < skipBelow)continue;+ var before = i > 0 ? text[i - 1] : -1;var codePoint = char.IsHighSurrogate(c) && i + 1 < text.Length && char.IsLowSurrogate(text[i + 1])? char.ConvertToUtf32(c, text[++i]): c;- if (classifier(codePoint) is { } script)+ var after = i + 1 < text.Length ? text[i + 1] : -1;+ if (classifier(codePoint, before, after) is { } script)return script;}return null;@@ -274,7 +287,7 @@ internal static partial class ComplexScripts/// here: the conversion then reports what is wrong with them)./// </summary>/// <param name="format">Format id of the file (docx, txt, hwpx, ...).</param>- public static string? FindInFile(string path, string format) => FindInFile(path, format, ScriptOf, Blocks[0].Start);+ public static string? FindInFile(string path, string format) => FindInFile(path, format, Shaping, Blocks[0].Start);/// <summary>/// The first script in the text of a document that rhwp draws but cannot measure (rhwp#8143), or null.@@ -284,7 +297,10 @@ internal static partial class ComplexScripts/// <summary>/// Both answers of the scan in one pass: the first script that needs shaping stops the scan and wins the- /// file; the first script rhwp cannot measure is remembered while the scan runs on.+ /// file; the first script rhwp cannot measure is remembered while the scan runs on. Only letters inside words+ /// count: a symbol or unit on its own (α, Ω, μm in a Korean report) draws fine, and rhwp, the most faithful+ /// renderer for Korean documents, keeps them. An extended Latin letter or combining mark counts next to any letter+ /// (Polish, Vietnamese words); a letter of another script only next to one of the same script (Отчёт, Αναφορά)./// </summary>/// <param name="format">Format id of the file (docx, txt, hwpx, ...).</param>public static (string? Shaping, string? Unmeasured) FindBothInFile(string path, string format)@@ -294,16 +310,28 @@ internal static partial class ComplexScriptsvar shaping = FindInFile(path, format, Both, skipBelow: 0x0100);return (shaping, unmeasured);- string? Both(int codePoint)+ string? Both(int codePoint, int before, int after){if (ScriptOf(codePoint) is { } script)return script;- unmeasured ??= UnmeasuredScriptOf(codePoint);+ if (unmeasured is null && UnmeasuredScriptOf(codePoint) is { } name && InWord(name, before, after))+ unmeasured = name;return null;}++ static bool InWord(string name, int before, int after)+ {+ if (before == Unknown || after == Unknown)+ return true;+ return name is "Latin Extended" or "IPA" or "combining marks"+ ? IsLetter(before) || IsLetter(after)+ : (before >= 0 && UnmeasuredScriptOf(before) == name) || (after >= 0 && UnmeasuredScriptOf(after) == name);+ }++ static bool IsLetter(int c) => c >= 0 && char.IsLetter((char)c);}- private static string? FindInFile(string path, string format, Func<int, string?> classifier, int skipBelow)+ private static string? FindInFile(string path, string format, Classifier classifier, int skipBelow){try{@@ -336,7 +364,7 @@ internal static partial class ComplexScripts/// <summary>Text files are read whole (as the readers do), up to this size; a larger one is scanned in its first part.</summary>private const int MaxTextBytes = 64 * 1024 * 1024;- private static string? FindInText(string path, Func<int, string?> classifier, int skipBelow)+ private static string? FindInText(string path, Classifier classifier, int skipBelow){byte[] bytes;using (var stream = File.OpenRead(path))@@ -347,7 +375,7 @@ internal static partial class ComplexScriptsreturn FindInString(DocumentReaders.DecodeText(bytes), classifier, skipBelow);}- private static string? FindInString(string text, Func<int, string?> classifier, int skipBelow)+ private static string? FindInString(string text, Classifier classifier, int skipBelow){if (Find(text, classifier, skipBelow) is { } script)return script;@@ -361,7 +389,7 @@ internal static partial class ComplexScriptsvalue += 0x10000; // RTF writes code points above 32767 as negative numbersif (value is < 0 or > 0x10FFFF)continue; // a reference can name a value outside Unicode (�, \u-99999): it is no script- if (classifier(value) is { } referenced)+ if (classifier(value, Unknown, Unknown) is { } referenced)return referenced;}return null;@@ -393,7 +421,7 @@ internal static partial class ComplexScripts};}- private static string? FindInPackage(string path, string format, Func<int, string?> classifier, int skipBelow)+ private static string? FindInPackage(string path, string format, Classifier classifier, int skipBelow){using var zip = ZipFile.OpenRead(path);var buffer = new char[64 * 1024];diff --git a/tests/Filee.Engines.Tests/ComplexScriptTests.cs b/tests/Filee.Engines.Tests/ComplexScriptTests.csindex 49d4389c..279f3ecf 100644--- a/tests/Filee.Engines.Tests/ComplexScriptTests.cs+++ b/tests/Filee.Engines.Tests/ComplexScriptTests.cs@@ -119,6 +119,38 @@ public class ComplexScriptTests(EngineFixture fx) : IClassFixture<EngineFixture>Assert.Null(ComplexScripts.FindInFile(Docx("russian-2.docx", "Отчёт"), "docx"));}+ [Fact]+ public void A_lone_symbol_keeps_rhwp()+ {+ // A Greek letter or a unit between spaces (α, Ω, μ in a Korean report) draws fine on its own: only letters+ // inside a word overlap, so only they send a document away from rhwp.+ var folder = fx.NewFolder();+ string Text(string name, string text)+ {+ var path = Path.Combine(folder, name);+ File.WriteAllText(path, text, new UTF8Encoding(false));+ return path;+ }++ Assert.Null(ComplexScripts.FindUnmeasuredInFile(Text("symbols.md", "각도 α = 30°, 저항 10 Ω, 두께 5 μm"), "md"));+ Assert.Equal("Greek", ComplexScripts.FindUnmeasuredInFile(Text("word.md", "결과: Αναφορά"), "md"));+ Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("short.md", "Да"), "md"));+ }++ [Fact]+ public void An_hwpx_stays_with_rhwp_for_unmeasured_letters()+ {+ // rhwp is the most faithful renderer for HWPX: Cyrillic in one only goes elsewhere when it needs shaping.+ var path = Path.Combine(fx.NewFolder(), "report.hwpx");+ using (var zip = System.IO.Compression.ZipFile.Open(path, System.IO.Compression.ZipArchiveMode.Create))+ {+ using var writer = new StreamWriter(zip.CreateEntry("Contents/section0.xml").Open());+ writer.Write("<hs:sec xmlns:hs=\"http://www.hancom.co.kr/hwpml/2011/section\"><hp:t xmlns:hp=\"x\">보고서 Отчёт</hp:t></hs:sec>");+ }+ Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(path, "hwpx"));+ Assert.Null(new RhwpConverter().ScanScripts(path, "hwpx").WithoutWidths);+ }+[Fact]public void A_character_reference_outside_unicode_is_skipped(){