Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.
C# audio-converteravaloniabatch-conversionconvertercsharpdocument-converterdocx-to-pdfdotnetdrag-and-dropfile-conversionfile-converterheic-converterhwphwpximage-converterlinuxmacospdf-convertervideo-converterwindows
// Text that needs shaping: Arabic, Hebrew, the Indic scripts, Thai, ... (issues #185 Gujarati and #202 Arabic).
// * Documents → PDF: rhwp draws such text unshaped, so a document with it goes to LibreOffice, or is refused with that
// reason when LibreOffice is missing (ComplexScripts finds the text, RhwpConverter's PDF edges are Unshaped).
// * PDF → DOCX / TXT: the reader takes ActualText, keeps marks with their letters in content-stream order, reads
// right-to-left words and lines from the right and marks such paragraphs right to left.
// The PDFs are written glyph by glyph (GlyphPdfBuilder) the way the shaping engines leave them.
using System.IO.Compression;
using System.Text;
using System.Xml.Linq;
using Filee.Core.Conversion;
using Filee.Core.Presets;
using Filee.Engines.Hwp;
using Filee.Engines.Infrastructure;
using Filee.Engines.Pdf;
using static Filee.Engines.Tests.DocxAssert;
namespace Filee.Engines.Tests;
public class ComplexScriptTests(EngineFixture fx) : IClassFixture<EngineFixture>
{
private static string PlainText(string pdf) => PdfDocumentReader.PlainText(PdfDocumentReader.ReadText(pdf)).Trim();
// ───────────────────────── Finding the text ─────────────────────────
[Fact]
public void Scripts_that_need_shaping_are_found_in_documents()
{
var folder = fx.NewFolder();
string Docx(string name, string text, string? styles = null)
{
var builder = new DocxBuilder();
if (styles is not null)
builder.Styles = styles;
builder.Paragraph(DocxBuilder.P(DocxBuilder.R(text)));
return builder.Save(Path.Combine(folder, name));
}
string Text(string name, string text)
{
var path = Path.Combine(folder, name);
File.WriteAllText(path, text, new UTF8Encoding(false));
return path;
}
Assert.Equal("Gujarati", ComplexScripts.FindInFile(Docx("gujarati.docx", "સૌરાષ્ટ્ર ઘરખેડ"), "docx"));
Assert.Equal("Arabic", ComplexScripts.FindInFile(Docx("arabic.docx", "Report: مرحبا بالعالم"), "docx"));
Assert.Null(ComplexScripts.FindInFile(Docx("plain.docx", "English, 한국어, 中文, Ελληνικά, Русский"), "docx"));
// Style and font names per script are no text of the document.
Assert.Null(ComplexScripts.FindInFile(Docx("styles.docx", "English only",
$"<w:styles {DocxBuilder.Namespaces}><w:style w:type=\"paragraph\" w:styleId=\"Title\"><w:name w:val=\"عنوان\"/></w:style></w:styles>"), "docx"));
Assert.Equal("Devanagari", ComplexScripts.FindInFile(Text("hindi.txt", "Title\nहिन्दी भाषा"), "txt"));
Assert.Equal("Hebrew", ComplexScripts.FindInFile(Text("hebrew.html", "<p>שלום</p>"), "html"));
Assert.Equal("Thai", ComplexScripts.FindInFile(Text("thai.rtf", @"{\rtf1 \u3585?\u3634?}"), "rtf"));
Assert.Null(ComplexScripts.FindInFile(Text("plain.md", "# Title\n\nC:\\users\\me and © 2026"), "md"));
Assert.Null(ComplexScripts.FindInFile(Text("binary.hwp", "ignored"), "hwp"));
// Web archives: the HTML part, quoted-printable here, as Chrome saves it.
const string Mhtml = "MIME-Version: 1.0\r\nContent-Type: multipart/related; boundary=\"b\"\r\n\r\n--b\r\n" +
"Content-Type: text/html; charset=utf-8\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\n" +
"<p>=E0=AA=B8=E0=AB=8C=E0=AA=B0=E0=AA=BE</p>\r\n--b--\r\n";
Assert.Equal("Gujarati", ComplexScripts.FindInFile(Text("page.mhtml", Mhtml), "mhtml"));
Assert.Null(ComplexScripts.FindInFile(Text("plain.mhtml", Mhtml.Replace("=E0=AA=B8=E0=AB=8C=E0=AA=B0=E0=AA=BE", "Hello")), "mhtml"));
// OFD: the text of the pages (an official document in Uyghur), not the resource names.
string Ofd(string name, string text)
{
var path = Path.Combine(folder, name);
using var zip = ZipFile.Open(path, ZipArchiveMode.Create);
using (var writer = new StreamWriter(zip.CreateEntry("Doc_0/Pages/Page_0/Content.xml").Open()))
writer.Write($"<ofd:Page xmlns:ofd=\"http://www.ofdspec.org/2016\"><ofd:TextCode X=\"0\" Y=\"4\">{text}</ofd:TextCode></ofd:Page>");
using (var writer = new StreamWriter(zip.CreateEntry("Doc_0/PublicRes.xml").Open()))
writer.Write("<Res><Font ID=\"1\" FontName=\"UKIJ Tuz مرحبا\"/></Res>");
return path;
}
Assert.Equal("Arabic", ComplexScripts.FindInFile(Ofd("uyghur.ofd", "ئۇيغۇرچە ھۆججەت"), "ofd"));
Assert.Null(ComplexScripts.FindInFile(Ofd("chinese.ofd", "电子发票(普通发票)"), "ofd"));
}
[Fact]
public void Scripts_without_widths_are_found_in_documents()
{
// rhwp#8143: rhwp's width tables cover only ASCII, Latin-1, general punctuation, math, the CJK forms and
// Hangul, so a letter of any other script — Polish, Turkish, Vietnamese, Greek, Cyrillic, ... — is drawn
// but laid out at a fixed half-em.
var folder = fx.NewFolder();
string Docx(string name, string text)
{
var builder = new DocxBuilder();
builder.Paragraph(DocxBuilder.P(DocxBuilder.R(text)));
return builder.Save(Path.Combine(folder, name));
}
string Text(string name, string text)
{
var path = Path.Combine(folder, name);
File.WriteAllText(path, text, new UTF8Encoding(false));
return path;
}
Assert.Equal("Latin Extended", ComplexScripts.FindUnmeasuredInFile(Docx("polish.docx", "Zażółć gęślą jaźń"), "docx"));
Assert.Equal("Latin Extended", ComplexScripts.FindUnmeasuredInFile(Text("vietnamese.md", "Tiếng Việt"), "md"));
Assert.Equal("Greek", ComplexScripts.FindUnmeasuredInFile(Docx("greek.docx", "Αναφορά Report"), "docx"));
Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Docx("russian.docx", "Отчёт за месяц"), "docx"));
Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("russian.txt", "Заголовок\nтекст"), "txt"));
Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("russian.md", "# Отчёт\n\ntext"), "md"));
Assert.Equal("Armenian", ComplexScripts.FindUnmeasuredInFile(Text("armenian.txt", "Հայերեն տեքստ"), "txt"));
Assert.Equal("Georgian", ComplexScripts.FindUnmeasuredInFile(Text("georgian.txt", "ქართული ტექსტი"), "txt"));
// A precomposed é is Latin-1 and measured; e + ´ decomposed is not.
Assert.Equal("combining marks", ComplexScripts.FindUnmeasuredInFile(Text("decomposed.md", "cafe\u0301"), "md"));
// Latin-1 letters are measured, Hangul, CJK and the full-width forms are estimated right at a full em,
// and text that needs shaping is the other scan's job.
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("latin1.docx", "Ça été über Straße niño"), "docx"));
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("korean.docx", "한국어 문서"), "docx"));
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("japanese.docx", "日本語のテキスト"), "docx"));
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("chinese.docx", "中文文档"), "docx"));
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("fullwidth.docx", "ABC"), "docx"));
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Docx("arabic.docx", "مرحبا بالعالم"), "docx"));
Assert.Null(ComplexScripts.FindInFile(Docx("russian-2.docx", "Отчёт"), "docx"));
}
[Fact]
public void A_lone_symbol_keeps_rhwp()
{
// A Greek letter or a unit between spaces (α, Ω, μ in a Korean report) draws fine on its own: only letters
// inside a word overlap, so only they send a document away from rhwp.
var folder = fx.NewFolder();
string Text(string name, string text)
{
var path = Path.Combine(folder, name);
File.WriteAllText(path, text, new UTF8Encoding(false));
return path;
}
Assert.Null(ComplexScripts.FindUnmeasuredInFile(Text("symbols.md", "각도 α = 30°, 저항 10 Ω, 두께 5 μm"), "md"));
Assert.Equal("Greek", ComplexScripts.FindUnmeasuredInFile(Text("word.md", "결과: Αναφορά"), "md"));
Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(Text("short.md", "Да"), "md"));
}
[Fact]
public void An_hwpx_stays_with_rhwp_for_unmeasured_letters()
{
// rhwp is the most faithful renderer for HWPX: Cyrillic in one only goes elsewhere when it needs shaping.
var path = Path.Combine(fx.NewFolder(), "report.hwpx");
using (var zip = System.IO.Compression.ZipFile.Open(path, System.IO.Compression.ZipArchiveMode.Create))
{
using var writer = new StreamWriter(zip.CreateEntry("Contents/section0.xml").Open());
writer.Write("<hs:sec xmlns:hs=\"http://www.hancom.co.kr/hwpml/2011/section\"><hp:t xmlns:hp=\"x\">보고서 Отчёт</hp:t></hs:sec>");
}
Assert.Equal("Cyrillic", ComplexScripts.FindUnmeasuredInFile(path, "hwpx"));
Assert.Null(new RhwpConverter().ScanScripts(path, "hwpx").WithoutWidths);
}
[Fact]
public void A_character_reference_outside_unicode_is_skipped()
{
// � (0x110000) and RTF \u-99999 (still negative past +0x10000) name no code point: a malformed
// reference before a document's letters must not lose both answers of the scan to an exception.
var folder = fx.NewFolder();
string Text(string name, string text)
{
var path = Path.Combine(folder, name);
File.WriteAllText(path, text, new UTF8Encoding(false));
return path;
}
var html = ComplexScripts.FindBothInFile(Text("bad-ref.html", "<p>� של</p>"), "html");
Assert.Equal("Hebrew", html.Shaping);
var rtf = ComplexScripts.FindBothInFile(Text("bad-ref.rtf", @"{\rtf1 \u-99999? ᒈ\u1513?\u1500?}"), "rtf");
Assert.Equal("Hebrew", rtf.Shaping);
}
[Fact]
public void Variation_selectors_are_not_unmeasured_text()
{
// U+FE0F picks an emoji's picture form (❤️, ✔️), U+E0100 an ideograph's registered variant (葛󠄀): the
// zero-width mark rides on a letter rhwp measures, so it must not reroute the document.
var folder = fx.NewFolder();
var builder = new DocxBuilder();
builder.Paragraph(DocxBuilder.P(DocxBuilder.R("Déjà vu ❤️ ✔️")));
var docx = builder.Save(Path.Combine(folder, "emoji.docx"));
var txt = Path.Combine(folder, "ivs.txt");
File.WriteAllText(txt, "葛\U000E0100の本文", new UTF8Encoding(false));
Assert.Null(ComplexScripts.FindBothInFile(docx, "docx").Unmeasured);
Assert.Null(ComplexScripts.FindBothInFile(txt, "txt").Unmeasured);
}
[Fact]
public void Both_answers_come_from_one_scan()
{
// Cyrillic before Arabic: the scan remembers the first unmeasured script while it looks on for shaping
// text, so the file keeps both answers.
var path = Path.Combine(fx.NewFolder(), "mixed.txt");
File.WriteAllText(path, "Отчёт مرحبا", new UTF8Encoding(false));
var (shaping, unmeasured) = ComplexScripts.FindBothInFile(path, "txt");
Assert.Equal("Arabic", shaping);
Assert.Equal("Cyrillic", unmeasured);
}
[Fact]
public async Task Scripts_are_found_in_hwpx_sections()
{
var folder = fx.NewFolder();
var txt = Path.Combine(folder, "tamil.txt");
await File.WriteAllTextAsync(txt, "தமிழ் மொழி\n한글 문단", TestContext.Current.CancellationToken);
var job = await fx.ConvertAsync([txt], new Preset { TargetFormat = "hwpx" });
Assert.Equal(JobState.Completed, job.State);
Assert.Equal("Tamil", ComplexScripts.FindInFile(job.Outputs.Single(), "hwpx"));
Assert.Null(ComplexScripts.FindInFile(await fx.MakeHwpxAsync(fx.NewFolder()), "hwpx"));
}
[Fact]
public void Rhwp_says_it_draws_pdf_text_unshaped()
{
var rhwp = fx.Converters.Single(c => c.Id == "rhwp");
Assert.All(rhwp.Edges.Where(e => e.To == "pdf"), e => Assert.True(e.Unshaped));
Assert.All(rhwp.Edges.Where(e => e.To != "pdf"), e => Assert.False(e.Unshaped));
Assert.IsAssignableFrom<ITextShapingCheck>(rhwp);
}
[Fact]
public void A_file_is_scanned_again_once_its_text_changed()
{
// A file converted twice in a session — a watch folder, a document saved again — is scanned again:
// an answer cached on the path would route its new text the old way.
var path = Path.Combine(fx.NewFolder(), "report.md");
File.WriteAllText(path, "A plain English report.", new UTF8Encoding(false));
var rhwp = new RhwpConverter();
var clean = rhwp.ScanScripts(path, "md");
Assert.Null(clean.NeedsShaping);
Assert.Null(clean.WithoutWidths);
File.WriteAllText(path, "مرحبا بالعالم", new UTF8Encoding(false));
Assert.Equal("Arabic", rhwp.ScanScripts(path, "md").NeedsShaping);
}
[Fact]
public async Task A_gujarati_document_goes_to_libreoffice_or_is_refused_with_the_reason()
{
var available = fx.Converters.Where(c => fx.Catalog.StatusOf(c).IsAvailable).Select(c => c.Id).ToHashSet();
Assert.SkipUnless(available.Contains("rhwp") || available.Contains("libreoffice"), "neither rhwp nor LibreOffice is available");
var builder = new DocxBuilder();
builder.Paragraph(DocxBuilder.P(DocxBuilder.R("સૌરાષ્ટ્ર ઘરખેડ ગણોત વહીવટ")));
var docx = builder.Save(Path.Combine(fx.NewFolder(), "gujarati.docx"));
var job = await fx.ConvertAsync([docx], new Preset { TargetFormat = "pdf" });
var file = Assert.Single(job.Files);
if (available.Contains("libreoffice"))
{
Assert.Equal(FileState.Done, file.State);
}
else
{
Assert.Equal(FileState.Failed, file.State);
Assert.Equal("error.needs_shaping", file.ErrorKey);
Assert.Equal("Gujarati", file.ErrorDetail);
}
}
// ───────────────────────── Reading PDFs ─────────────────────────
/// <summary>
/// Text laid out like rhwp did: one glyph per character, a placeholder glyph (a dotted circle, here mapped to
/// <paramref name="placeholder"/>) before every mark, bases overlapping the marks.
/// </summary>
private static void Unshaped(GlyphPdfBuilder pdf, string text, double x, double y, string placeholder, double size = 14)
{
foreach (var word in text.Split(' '))
{
foreach (var c in word)
{
if (char.GetUnicodeCategory(c) is System.Globalization.UnicodeCategory.NonSpacingMark or System.Globalization.UnicodeCategory.SpacingCombiningMark)
{
pdf.Glyph(placeholder, x, y, 0.6 * size, size);
x += 0.6 * size;
var spacing = char.GetUnicodeCategory(c) == System.Globalization.UnicodeCategory.SpacingCombiningMark;
pdf.Glyph(c.ToString(), x, y, spacing ? 0.27 * size : 0, size);
x += spacing ? 0.27 * size : 0;
}
else
{
pdf.Glyph(c.ToString(), x - 0.1 * size, y, 0.55 * size, size);
x += 0.45 * size;
}
}
x += 1.2 * size;
}
}
[Fact]
public void Text_drawn_without_shaping_is_read_in_its_logical_order()
{
// Issue #185's PDF: Gujarati from rhwp, its dotted circles mapped to U+0002; Hindi with them mapped to a vowel sign.
var pdf = new GlyphPdfBuilder();
Unshaped(pdf, "સૌરાષ્ટ્ર ઘરખેડ ગણોત", 72, 100, "\u0002");
Unshaped(pdf, "भाषा में दस्तावेज़", 72, 140, "\u093F");
var path = pdf.Save(Path.Combine(fx.NewFolder(), "unshaped.pdf"));
Assert.Equal("સૌરાષ્ટ્ર ઘરખેડ ગણોત\r\n\r\nभाषा में दस्तावेज़", PlainText(path));
}
[Fact]
public void Actual_text_replaces_the_glyphs_it_covers()
{
// LibreOffice: a conjunct drawn as one glyph and a vowel sign drawn before its consonant carry their text.
var pdf = new GlyphPdfBuilder();
var x = 72.0;
pdf.Glyph("ર", x, 100, 7).Glyph("\u0ABE", x += 7, 100, 4);
pdf.ActualText("ષ્ટ્ર", p => p.Glyph("ષ", x += 4, 100, 12));
x += 12 + 4;
pdf.ActualText("નિ", p => p.Glyph("\u0ABF", x, 100, 3).Glyph("ન", x + 3, 100, 8));
pdf.Glyph("ય", x += 11, 100, 8).Glyph("મ", x + 8, 100, 9);
// Latin text keeps working on the same page, with an ActualText for a ligature.
pdf.Run("Office ", 72, 130, 6);
pdf.ActualText("fi", p => p.Glyph("fi", 72 + 7 * 6, 130, 6));
pdf.Run("le", 72 + 8 * 6, 130, 6);
var path = pdf.Save(Path.Combine(fx.NewFolder(), "actualtext.pdf"));
Assert.Equal("રાષ્ટ્ર નિયમ\r\n\r\nOffice file", PlainText(path));
}
[Fact]
public void Vowel_signs_drawn_before_their_consonant_follow_it()
{
// Word's PDFs keep the visual order of ि and give no ActualText: ि क → कि.
var pdf = new GlyphPdfBuilder();
var x = 72.0;
pdf.Glyph("\u093F", x, 100, 3).Glyph("क", x + 3, 100, 8).Glyph("स", x + 11, 100, 8).Glyph("\u0940", x + 19, 100, 3);
x += 30;
pdf.Glyph("\u093F", x, 100, 3).Glyph("ह", x + 3, 100, 7).Glyph("न्", x + 10, 100, 5).Glyph("द", x + 15, 100, 7).Glyph("\u0940", x + 22, 100, 3);
var path = pdf.Save(Path.Combine(fx.NewFolder(), "prebase.pdf"));
Assert.Equal("किसी हिन्दी", PlainText(path));
}
/// <summary>
/// A right-to-left line as Word and LibreOffice draw it: words from the right edge leftwards, the glyphs of each
/// word left to right (so an Arabic word's letters are reversed in the stream), numbers left to right.
/// </summary>
/// <returns>The left edge of the last word.</returns>
private static double RightToLeft(GlyphPdfBuilder pdf, IEnumerable<string> words, double right, double y, double advance = 6)
{
var x = right + advance;
foreach (var word in words)
{
x -= advance; // the space between words
var visual = ComplexScripts.Direction(word) < 0 ? new string([.. word.Reverse()]) : word;
x -= visual.Length * advance;
pdf.Run(visual, x, y, advance);
}
return x;
}
[Fact]
public async Task Arabic_is_read_from_right_to_left_into_a_right_to_left_paragraph()
{
// Issue #202: Arabic PDF → DOCX. A two-line paragraph with a number and an English word in it, a short one with
// harakat (marks after their letter in the stream), then an English paragraph.
var pdf = new GlyphPdfBuilder();
RightToLeft(pdf, ["هذا", "نص", "عربي", "للاختبار", "مع", "كلمة", "English", "في", "الوسط", "والرقم", "وهذه", "جملة", "طويلة", "جدا"], 523, 100);
// The second line ends the sentence: "2024." shows as ".2024" left of the last Arabic word.
var end = RightToLeft(pdf, ["اللغة", "العربية", "جميلة", "2024"], 523, 115);
pdf.Glyph(".", end - 6, 115, 6);
// مُحَمَّد in visual order: د, م with fatha and shadda, ح with fatha, م with damma.
pdf.Glyph("د", 470, 160, 6).Glyph("م", 476, 160, 6).Glyph("\u064E", 476, 160, 0).Glyph("\u0651", 476, 160, 0)
.Glyph("ح", 482, 160, 6).Glyph("\u064E", 482, 160, 0).Glyph("م", 488, 160, 6).Glyph("\u064F", 488, 160, 0);
pdf.Run("Plain English paragraph.", 72, 200, 6);
var path = pdf.Save(Path.Combine(fx.NewFolder(), "arabic.pdf"));
Assert.Equal(
"هذا نص عربي للاختبار مع كلمة English في الوسط والرقم وهذه جملة طويلة جدا اللغة العربية جميلة 2024.\r\n\r\n" +
"مُحَمَّد\r\n\r\nPlain English paragraph.", PlainText(path));
var job = await fx.ConvertAsync([path], new Preset { TargetFormat = "docx" });
Assert.Equal(JobState.Completed, job.State);
var docx = job.Outputs.Single();
ValidPackage(docx);
var paragraphs = Body(docx).Descendants(W + "p").Where(p => p.Descendants(W + "t").Any()).ToList();
Assert.Equal(3, paragraphs.Count);
bool Bidi(XElement p) => p.Element(W + "pPr")?.Element(W + "bidi") is not null;
Assert.Equal([true, true, false], paragraphs.Select(Bidi));
// Arabic runs are right to left (Word then uses their complex script font and size), English ones are not.
var runs = paragraphs[0].Descendants(W + "r").Where(r => r.Element(W + "t") is not null).ToList();
Assert.All(runs.Where(r => ComplexScripts.Direction(r.Element(W + "t")!.Value) < 0), r => Assert.NotNull(r.Element(W + "rPr")?.Element(W + "rtl")));
Assert.Empty(paragraphs[2].Descendants(W + "rtl"));
}
}