Skip to content

armgit / Filee

Public Imported from GitHub
0

Free, open-source file converter. Convert 263 formats offline by drag & drop — images, PDF, Word, Excel, PowerPoint, HWP/HWPX, video, audio, e-books and archives. No upload, no Office needed.

PDF reader: no space between two Chinese or Japanese words (#691)

KnifeLemonbf691172d99a
diff --git a/src/Filee.Engines/Pdf/PdfDocumentReader.cs b/src/Filee.Engines/Pdf/PdfDocumentReader.cs
index e3fbe1d0..ac3436bc 100644
--- a/src/Filee.Engines/Pdf/PdfDocumentReader.cs
+++ b/src/Filee.Engines/Pdf/PdfDocumentReader.cs
@@ -821,7 +821,8 @@ internal sealed partial class PdfDocumentReader
}
/// <summary>
- /// Words joined into runs of equal formatting; line ends become spaces (not inside CJK text or after a hyphen), or
+ /// Words joined into runs of equal formatting, with a space between words (not between two CJK words); line ends
+ /// become spaces (not inside CJK text or after a hyphen), or
/// line breaks where the line ended early on purpose. Tabs of a tabbed row stay tabs.
/// </summary>
private static void Runs(Item item, List<HInline> output, bool rightToLeft)
@@ -845,7 +846,10 @@ internal sealed partial class PdfDocumentReader
}
else if (w > 0)
{
- pieces[^1] = (previousText + " ", previousFormat, previousLink);
+ // Not between two Chinese or Japanese words: an OCR layer (Tesseract's) puts a space after
+ // every word it found, also inside a sentence of Han characters (#691). Korean keeps them.
+ if (!(IsCjk(previousText[^1]) && IsCjk(text[0])))
+ pieces[^1] = (previousText + " ", previousFormat, previousLink);
}
else if (previousText.EndsWith('-') && previousText.Length > 1 && char.IsLetter(previousText[^2]) && text.Length > 0 && char.IsLower(text[0]))
{
diff --git a/tests/Filee.Engines.Tests/VisionOcrTests.cs b/tests/Filee.Engines.Tests/VisionOcrTests.cs
index a5b57d4e..424a957a 100644
--- a/tests/Filee.Engines.Tests/VisionOcrTests.cs
+++ b/tests/Filee.Engines.Tests/VisionOcrTests.cs
@@ -136,6 +136,33 @@ public class OcrTextLayerTests(EngineFixture fx) : IClassFixture<EngineFixture>
var text = await File.ReadAllTextAsync(job.Outputs.Single(), TestContext.Current.CancellationToken);
Assert.Contains("The quarterly report is ready", text);
}
+
+ [Fact]
+ public async Task Chinese_and_japanese_words_of_the_layer_join_without_spaces_and_korean_keeps_them()
+ {
+ // Tesseract writes a space after every word, also between the Han characters of one sentence (#691).
+ var folder = fx.NewFolder();
+ var picture = VisionFakes.Picture(folder, "scan.png", 600, 800);
+ var pdf = Path.Combine(folder, "scan.pdf");
+ using (var writer = new OcrPdfWriter())
+ {
+ writer.AddPage(picture, 144, 192,
+ [
+ OcrLine.Estimate("本 书 是 为 USB 入门 和 提高 者 编写 。", 0.1, 0.8, 0.8, 0.04),
+ OcrLine.Estimate("これ は テスト です", 0.1, 0.6, 0.8, 0.04),
+ OcrLine.Estimate("검색 가능한 문서", 0.1, 0.4, 0.8, 0.04),
+ ]);
+ writer.Save(pdf);
+ }
+
+ var job = await fx.ConvertAsync([pdf], new Preset { TargetFormat = "txt" });
+
+ Assert.True(job.State == JobState.Completed, string.Join("; ", job.Files.Select(f => $"{f.ErrorKey} {f.ErrorDetail}")));
+ var text = await File.ReadAllTextAsync(job.Outputs.Single(), TestContext.Current.CancellationToken);
+ Assert.Contains("本书是为 USB 入门和提高者编写。", text);
+ Assert.Contains("これはテストです", text);
+ Assert.Contains("검색 가능한 문서", text);
+ }
}
public class VisionConverterTests(EngineFixture fx) : IClassFixture<EngineFixture>