Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(pdf): rejoin line-break hyphens / strip U+FFFD in Go layout (temporary workaround) (#20274)

xugangqiang02b668c1de45
diff --git a/internal/deepdoc/parser/pdf/layout/dehyphen.go b/internal/deepdoc/parser/pdf/layout/dehyphen.go
new file mode 100644
index 0000000000..8de67ed692
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/dehyphen.go
@@ -0,0 +1,171 @@
+package layout
+
+import (
+ "math"
+ "regexp"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// garbageHyphenRE removes U+FFFD (replacement char) and U+00AD (soft hyphen)
+// — both are pure extraction garbage, never real content — along with the
+// optional single space a line break inserts after them, so the two halves of
+// a hyphenated word rejoin ("pre� cipitation" → "precipitation").
+var garbageHyphenRE = regexp.MustCompile(`[` + "\uFFFD\u00AD" + `]\s?`)
+
+// lineBreakHyphenRE rejoins a real hyphen that a PDF placed at a line break
+// ("re- covery" → "recovery"). It fires only when a space follows the hyphen,
+// so in-word hyphens without a space ("lithium-ion", "well-known") survive.
+var lineBreakHyphenRE = regexp.MustCompile(`(\p{L})[` + "-\u2010\u2011\u2043" + `]\s(\p{L})`)
+
+// Dehyphenate rejoins words that a PDF split across a line break with a
+// hyphen. Some PDFs (e.g. Elsevier journals set in CharisSIL) encode the
+// line-break hyphen with a glyph whose Unicode mapping the extractor fails to
+// resolve, so the boundary shows up as U+FFFD (replacement char) or U+00AD
+// (soft hyphen); others use a literal '-'. In every case the two halves land
+// in two vertically-adjacent text boxes and the later vertical merge glues
+// them with a space, yielding "pre- cipitation" or "pre� cipitation" instead
+// of "precipitation".
+//
+// Dehyphenate must run AFTER FinalReadingOrderMerge (so consecutive lines are
+// adjacent in the slice and share a column id) and BEFORE NaiveVerticalMerge
+// (which would otherwise insert the offending space). It collapses a box that
+// ends in a broken-hyphen marker with the following box when that box continues
+// the word (starts with a lowercase letter and is the next physical line).
+//
+// It also strips stray U+FFFD / U+00AD markers embedded inside a word (the
+// "text cleaning" half of the fix) so a split that happens to survive inside a
+// single box is still repaired.
+func Dehyphenate(boxes []pdf.TextBox) []pdf.TextBox {
+ out := make([]pdf.TextBox, 0, len(boxes))
+ for i := 0; i < len(boxes); i++ {
+ b := boxes[i]
+ // Rejoin a line-broken word first (the broken-hyphen marker must
+ // still be present on b's end for canDehyphenate to detect it).
+ if i+1 < len(boxes) && canDehyphenate(b, boxes[i+1]) {
+ b = joinBrokenHyphen(b, boxes[i+1])
+ i++ // consume the continuation box
+ }
+ // Then strip any remaining replacement/soft-hyphen garbage (e.g. a
+ // broken hyphen that landed mid-box with a line-break space).
+ b.Text = cleanIntraWordHyphens(b.Text)
+ out = append(out, b)
+ }
+ return out
+}
+
+// canDehyphenate reports whether prev ends in a broken-hyphen marker and curr
+// is the next line continuing the same word.
+func canDehyphenate(prev, curr pdf.TextBox) bool {
+ if prev.PageNumber != curr.PageNumber {
+ return false
+ }
+ if prev.ColID != curr.ColID {
+ return false
+ }
+ if !endsWithBrokenHyphen(prev.Text) {
+ return false
+ }
+ if !startsWithWordContinuation(curr.Text) {
+ return false
+ }
+ return linesAreVerticallyAdjacent(prev, curr)
+}
+
+// endsWithBrokenHyphen reports whether s ends in a broken-hyphen marker that
+// is preceded by at least one letter (so a lone "-" is not treated as a break).
+func endsWithBrokenHyphen(s string) bool {
+ s = strings.TrimRight(s, " \t")
+ if s == "" {
+ return false
+ }
+ r, _ := utf8.DecodeLastRuneInString(s)
+ if !isBrokenHyphenRune(r) {
+ return false
+ }
+ before := s[:len(s)-utf8.RuneLen(r)]
+ rb, _ := utf8.DecodeLastRuneInString(before)
+ return unicode.IsLetter(rb)
+}
+
+// stripTrailingBrokenHyphen removes a single trailing broken-hyphen marker.
+func stripTrailingBrokenHyphen(s string) string {
+ s = strings.TrimRight(s, " \t")
+ if s == "" {
+ return s
+ }
+ r, size := utf8.DecodeLastRuneInString(s)
+ if isBrokenHyphenRune(r) {
+ return s[:len(s)-size]
+ }
+ return s
+}
+
+// startsWithWordContinuation reports whether s begins with a lowercase letter.
+// A new sentence starts with an uppercase letter, so a lowercase start strongly
+// implies the previous line's hyphen was a real break rather than a dash.
+func startsWithWordContinuation(s string) bool {
+ s = strings.TrimLeft(s, " \t")
+ if s == "" {
+ return false
+ }
+ r, _ := utf8.DecodeRuneInString(s)
+ return unicode.IsLetter(r) && unicode.IsLower(r)
+}
+
+// linesAreVerticallyAdjacent reports whether curr is roughly the next physical
+// line below prev (not a distant paragraph). It is self-contained: the allowed
+// gap scales with the box heights so no external median height is needed.
+func linesAreVerticallyAdjacent(prev, curr pdf.TextBox) bool {
+ if curr.Top <= prev.Top {
+ return false
+ }
+ gap := curr.Top - prev.Bottom
+ lineH := math.Max(prev.Bottom-prev.Top, curr.Bottom-curr.Top)
+ if lineH <= 0 {
+ lineH = 1
+ }
+ return gap <= lineH*2.5
+}
+
+// joinBrokenHyphen concatenates curr onto prev after dropping prev's trailing
+// broken-hyphen marker, and expands prev's geometry to cover both lines.
+func joinBrokenHyphen(prev, curr pdf.TextBox) pdf.TextBox {
+ prev.Text = stripTrailingBrokenHyphen(prev.Text) + curr.Text
+ prev.Top = math.Min(prev.Top, curr.Top)
+ prev.Bottom = math.Max(prev.Bottom, curr.Bottom)
+ prev.X0 = math.Min(prev.X0, curr.X0)
+ prev.X1 = math.Max(prev.X1, curr.X1)
+ return prev
+}
+
+// cleanIntraWordHyphens repairs broken-hyphen garbage inside a single box's
+// text. U+FFFD / U+00AD are pure extraction noise and are deleted (with the
+// optional trailing line-break space) so the word halves rejoin; a real hyphen
+// at a line break ("re- covery") is rejoined the same way, while an in-word
+// hyphen without a following space ("lithium-ion", "well-known") is kept.
+func cleanIntraWordHyphens(s string) string {
+ if !strings.ContainsRune(s, '\uFFFD') && !strings.ContainsRune(s, '\u00AD') && !strings.ContainsRune(s, '-') {
+ return s
+ }
+ s = garbageHyphenRE.ReplaceAllString(s, "")
+ s = lineBreakHyphenRE.ReplaceAllString(s, "$1$2")
+ return s
+}
+
+// isBrokenHyphenRune reports whether r is a hyphen-like codepoint that a PDF
+// may use to split a word across a line break.
+func isBrokenHyphenRune(r rune) bool {
+ switch r {
+ case '\uFFFD', // replacement char (unresolved glyph)
+ '\u00AD', // soft hyphen
+ '-', // hyphen-minus
+ '\u2010', // hyphen
+ '\u2011': // non-breaking hyphen
+ return true
+ }
+ return false
+}
diff --git a/internal/deepdoc/parser/pdf/layout/dehyphen_test.go b/internal/deepdoc/parser/pdf/layout/dehyphen_test.go
new file mode 100644
index 0000000000..e357f0ef18
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/layout/dehyphen_test.go
@@ -0,0 +1,191 @@
+package layout
+
+import (
+ "strings"
+ "testing"
+
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// mkHyphenBox builds a TextBox with the given text on a single line at the given
+// top, with an approximate 10pt height.
+func mkHyphenBox(text string, top, colID, page int) pdf.TextBox {
+ return pdf.TextBox{
+ Text: text,
+ Top: float64(top),
+ Bottom: float64(top) + 10,
+ X0: 100,
+ X1: 100 + float64(len(text)*5),
+ ColID: colID,
+ PageNumber: page,
+ }
+}
+
+func TestDehyphenateFFFD(t *testing.T) {
+ // "pre" + U+FFFD at line end, "cipitation" continues on next line.
+ in := []pdf.TextBox{
+ mkHyphenBox("pre\ufffd", 100, 1, 1),
+ mkHyphenBox("cipitation", 112, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 {
+ t.Fatalf("expected 1 box, got %d: %+v", len(out), out)
+ }
+ if out[0].Text != "precipitation" {
+ t.Errorf("expected \"precipitation\", got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenateSoftHyphen(t *testing.T) {
+ in := []pdf.TextBox{
+ mkHyphenBox("sepa\u00ad", 100, 1, 1),
+ mkHyphenBox("ration", 112, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "separation" {
+ t.Errorf("expected rejoined \"separation\", got %+v", out)
+ }
+}
+
+func TestDehyphenateASCIIHyphen(t *testing.T) {
+ in := []pdf.TextBox{
+ mkHyphenBox("re-", 100, 1, 1),
+ mkHyphenBox("covery", 112, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "recovery" {
+ t.Errorf("expected rejoined \"recovery\", got %+v", out)
+ }
+}
+
+func TestDehyphenateNoMergeUppercaseContinuation(t *testing.T) {
+ // Line ends with '-' but the next line starts with a capital → new
+ // sentence / dash, not a hyphenated word. Must NOT merge.
+ in := []pdf.TextBox{
+ mkHyphenBox("word-", 100, 1, 1),
+ mkHyphenBox("Word", 112, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 2 {
+ t.Fatalf("expected 2 boxes (no merge), got %d: %+v", len(out), out)
+ }
+ if out[0].Text != "word-" || out[1].Text != "Word" {
+ t.Errorf("boxes changed unexpectedly: %q / %q", out[0].Text, out[1].Text)
+ }
+}
+
+func TestDehyphenateNoMergeDifferentColumn(t *testing.T) {
+ in := []pdf.TextBox{
+ mkHyphenBox("pre\ufffd", 100, 1, 1),
+ mkHyphenBox("cipitation", 112, 2, 1), // different column
+ }
+ out := Dehyphenate(in)
+ if len(out) != 2 {
+ t.Errorf("expected 2 boxes (different columns), got %d", len(out))
+ }
+}
+
+func TestDehyphenateNoMergeDifferentPage(t *testing.T) {
+ in := []pdf.TextBox{
+ mkHyphenBox("pre\ufffd", 100, 1, 1),
+ mkHyphenBox("cipitation", 112, 1, 2), // different page
+ }
+ out := Dehyphenate(in)
+ if len(out) != 2 {
+ t.Errorf("expected 2 boxes (different pages), got %d", len(out))
+ }
+}
+
+func TestDehyphenateNoMergeNonAdjacentLines(t *testing.T) {
+ // Marker present and lowercase continuation, but lines are far apart.
+ in := []pdf.TextBox{
+ mkHyphenBox("pre\ufffd", 100, 1, 1),
+ mkHyphenBox("cipitation", 300, 1, 1), // huge vertical gap
+ }
+ out := Dehyphenate(in)
+ if len(out) != 2 {
+ t.Errorf("expected 2 boxes (non-adjacent lines), got %d", len(out))
+ }
+}
+
+func TestDehyphenateCleanIntraWordGarbage(t *testing.T) {
+ // A replacement char embedded between two letters inside one box.
+ in := []pdf.TextBox{
+ mkHyphenBox("a\ufffdb", 100, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "ab" {
+ t.Errorf("expected \"ab\", got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenatePreservesRealHyphen(t *testing.T) {
+ // A literal hyphen inside a single word ("lithium-ion") must survive.
+ in := []pdf.TextBox{
+ mkHyphenBox("lithium-ion", 100, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "lithium-ion" {
+ t.Errorf("expected \"lithium-ion\" preserved, got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenateWellKnownPreserved(t *testing.T) {
+ // "well-known" has a real in-word hyphen with no following space.
+ in := []pdf.TextBox{
+ mkHyphenBox("well-known", 100, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "well-known" {
+ t.Errorf("expected \"well-known\" preserved, got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenateMidBoxFFFDWithSpace(t *testing.T) {
+ // Real-PDF shape: FFFD mid-box followed by the line-break space, then the
+ // continuation word (e.g. "devel� inorganic").
+ in := []pdf.TextBox{
+ mkHyphenBox("devel\ufffd inorganic", 100, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "develinorganic" {
+ t.Errorf("expected \"develinorganic\", got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenateLineBreakASCIIHyphenWithSpace(t *testing.T) {
+ // "re-" at a line break followed by the line-break space and "covery".
+ in := []pdf.TextBox{
+ mkHyphenBox("re- covery", 100, 1, 1),
+ }
+ out := Dehyphenate(in)
+ if len(out) != 1 || out[0].Text != "recovery" {
+ t.Errorf("expected \"recovery\", got %q", out[0].Text)
+ }
+}
+
+func TestDehyphenateWholePageAssembly(t *testing.T) {
+ // Two independent hyphenated breaks plus an unrelated line.
+ in := []pdf.TextBox{
+ mkHyphenBox("pre\ufffd", 100, 1, 1),
+ mkHyphenBox("cipitation", 112, 1, 1),
+ mkHyphenBox("sepa\ufffd", 130, 1, 1),
+ mkHyphenBox("ration", 142, 1, 1),
+ mkHyphenBox("lithium-ion", 160, 1, 1),
+ }
+ out := Dehyphenate(in)
+ var sb strings.Builder
+ for _, b := range out {
+ sb.WriteString(b.Text)
+ sb.WriteString(" ")
+ }
+ got := sb.String()
+ for _, want := range []string{"precipitation", "separation", "lithium-ion"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("assembled text %q missing %q", got, want)
+ }
+ }
+ if strings.Contains(got, "\ufffd") {
+ t.Errorf("assembled text still contains U+FFFD: %q", got)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.go
index a5af099654..663a25c5a1 100644
--- a/internal/deepdoc/parser/pdf/parser.go
+++ b/internal/deepdoc/parser/pdf/parser.go
@@ -592,6 +592,12 @@ func (p *Parser) buildLayout(ctx context.Context,
// page-leading title isolated in its own column.
boxes = lyt.FinalReadingOrderMerge(boxes)
+ // Rejoin words a PDF split across a line break with a hyphen. Must run
+ // after FinalReadingOrderMerge (consecutive lines are adjacent, same
+ // column) and before NaiveVerticalMerge (which would otherwise glue the
+ // halves with a space). See layout.Dehyphenate.
+ boxes = lyt.Dehyphenate(boxes)
+
boxes = lyt.NaiveVerticalMerge(boxes, medianHeights, medianWidths, pageEnglish)
result.Metrics.BoxesVertMerge = len(boxes)
if err := ctx.Err(); err != nil {
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.go b/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.go
new file mode 100644
index 0000000000..b8ddd3b09e
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.go
@@ -0,0 +1,80 @@
+//go:build manual
+
+package pdfoxide
+
+import (
+ "os"
+ "strings"
+ "testing"
+
+ "ragflow/internal/deepdoc/parser/pdf/layout"
+ pdf "ragflow/internal/deepdoc/parser/pdf/type"
+)
+
+// TestDehyphenateRealPDF is a regression test against the Elsevier article
+// that emits U+FFFD at every line-break hyphen (pre-cipitation, sepa-ration,
+// re-covery, ...). It is manual-tier and gated on RAGFLOW_TEST_PDF so it
+// never runs in CI and does not require committing the 1.5 MB PDF.
+//
+// Run locally with:
+//
+// RAGFLOW_TEST_PDF=~/Downloads/1-s2.0-S0956053X25001680-main.pdf \
+// bash build.sh --test-manual ./internal/deepdoc/parser/pdf/pdfoxide/ -run TestDehyphenateRealPDF -v
+func TestDehyphenateRealPDF(t *testing.T) {
+ path := os.Getenv("RAGFLOW_TEST_PDF")
+ if path == "" {
+ t.Skip("set RAGFLOW_TEST_PDF to the Elsevier PDF to run this regression test")
+ }
+
+ doc, err := Open(path)
+ if err != nil {
+ t.Fatalf("open %s: %v", path, err)
+ }
+ defer doc.Close()
+
+ chars, err := doc.GetPageChars(0)
+ if err != nil {
+ t.Fatalf("get page chars: %v", err)
+ }
+
+ // Convert the extractor's local char type into the layout TextChar type.
+ tchars := make([]pdf.TextChar, len(chars))
+ for i, c := range chars {
+ tchars[i] = pdf.TextChar{
+ X0: c.X0,
+ X1: c.X1,
+ Top: c.Top,
+ Bottom: c.Bottom,
+ Text: c.Text,
+ PageNumber: c.PageNumber,
+ }
+ }
+
+ boxes := layout.CharsToBoxes(tchars, 1, false)
+ boxes = layout.Dehyphenate(boxes)
+
+ var sb strings.Builder
+ for _, b := range boxes {
+ sb.WriteString(b.Text)
+ sb.WriteString("\n")
+ }
+ text := sb.String()
+
+ if strings.Contains(text, "\ufffd") {
+ t.Errorf("output still contains U+FFFD replacement char:\n%s", text)
+ }
+
+ // The three words the user reported must be rejoined, not broken.
+ for _, want := range []string{"precipitation", "separation", "recovery"} {
+ if !strings.Contains(text, want) {
+ t.Errorf("expected rejoined word %q in output, got:\n%s", want, text)
+ }
+ }
+
+ // And the broken forms must be gone.
+ for _, bad := range []string{"pre- cipitation", "pre\u00ad", "sepa\u00ad", "re\u00ad"} {
+ if strings.Contains(text, bad) {
+ t.Errorf("broken hyphenation not cleaned, found %q in:\n%s", bad, text)
+ }
+ }
+}