RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(pdf): rejoin line-break hyphens / strip U+FFFD in Go layout (temporary workaround) (#20274)
xugangqiang02b668c1de45
diff --git a/internal/deepdoc/parser/pdf/layout/dehyphen.go b/internal/deepdoc/parser/pdf/layout/dehyphen.gonew file mode 100644index 0000000000..8de67ed692--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/dehyphen.go@@ -0,0 +1,171 @@+package layout++import (+ "math"+ "regexp"+ "strings"+ "unicode"+ "unicode/utf8"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// garbageHyphenRE removes U+FFFD (replacement char) and U+00AD (soft hyphen)+// — both are pure extraction garbage, never real content — along with the+// optional single space a line break inserts after them, so the two halves of+// a hyphenated word rejoin ("pre� cipitation" → "precipitation").+var garbageHyphenRE = regexp.MustCompile(`[` + "\uFFFD\u00AD" + `]\s?`)++// lineBreakHyphenRE rejoins a real hyphen that a PDF placed at a line break+// ("re- covery" → "recovery"). It fires only when a space follows the hyphen,+// so in-word hyphens without a space ("lithium-ion", "well-known") survive.+var lineBreakHyphenRE = regexp.MustCompile(`(\p{L})[` + "-\u2010\u2011\u2043" + `]\s(\p{L})`)++// Dehyphenate rejoins words that a PDF split across a line break with a+// hyphen. Some PDFs (e.g. Elsevier journals set in CharisSIL) encode the+// line-break hyphen with a glyph whose Unicode mapping the extractor fails to+// resolve, so the boundary shows up as U+FFFD (replacement char) or U+00AD+// (soft hyphen); others use a literal '-'. In every case the two halves land+// in two vertically-adjacent text boxes and the later vertical merge glues+// them with a space, yielding "pre- cipitation" or "pre� cipitation" instead+// of "precipitation".+//+// Dehyphenate must run AFTER FinalReadingOrderMerge (so consecutive lines are+// adjacent in the slice and share a column id) and BEFORE NaiveVerticalMerge+// (which would otherwise insert the offending space). It collapses a box that+// ends in a broken-hyphen marker with the following box when that box continues+// the word (starts with a lowercase letter and is the next physical line).+//+// It also strips stray U+FFFD / U+00AD markers embedded inside a word (the+// "text cleaning" half of the fix) so a split that happens to survive inside a+// single box is still repaired.+func Dehyphenate(boxes []pdf.TextBox) []pdf.TextBox {+ out := make([]pdf.TextBox, 0, len(boxes))+ for i := 0; i < len(boxes); i++ {+ b := boxes[i]+ // Rejoin a line-broken word first (the broken-hyphen marker must+ // still be present on b's end for canDehyphenate to detect it).+ if i+1 < len(boxes) && canDehyphenate(b, boxes[i+1]) {+ b = joinBrokenHyphen(b, boxes[i+1])+ i++ // consume the continuation box+ }+ // Then strip any remaining replacement/soft-hyphen garbage (e.g. a+ // broken hyphen that landed mid-box with a line-break space).+ b.Text = cleanIntraWordHyphens(b.Text)+ out = append(out, b)+ }+ return out+}++// canDehyphenate reports whether prev ends in a broken-hyphen marker and curr+// is the next line continuing the same word.+func canDehyphenate(prev, curr pdf.TextBox) bool {+ if prev.PageNumber != curr.PageNumber {+ return false+ }+ if prev.ColID != curr.ColID {+ return false+ }+ if !endsWithBrokenHyphen(prev.Text) {+ return false+ }+ if !startsWithWordContinuation(curr.Text) {+ return false+ }+ return linesAreVerticallyAdjacent(prev, curr)+}++// endsWithBrokenHyphen reports whether s ends in a broken-hyphen marker that+// is preceded by at least one letter (so a lone "-" is not treated as a break).+func endsWithBrokenHyphen(s string) bool {+ s = strings.TrimRight(s, " \t")+ if s == "" {+ return false+ }+ r, _ := utf8.DecodeLastRuneInString(s)+ if !isBrokenHyphenRune(r) {+ return false+ }+ before := s[:len(s)-utf8.RuneLen(r)]+ rb, _ := utf8.DecodeLastRuneInString(before)+ return unicode.IsLetter(rb)+}++// stripTrailingBrokenHyphen removes a single trailing broken-hyphen marker.+func stripTrailingBrokenHyphen(s string) string {+ s = strings.TrimRight(s, " \t")+ if s == "" {+ return s+ }+ r, size := utf8.DecodeLastRuneInString(s)+ if isBrokenHyphenRune(r) {+ return s[:len(s)-size]+ }+ return s+}++// startsWithWordContinuation reports whether s begins with a lowercase letter.+// A new sentence starts with an uppercase letter, so a lowercase start strongly+// implies the previous line's hyphen was a real break rather than a dash.+func startsWithWordContinuation(s string) bool {+ s = strings.TrimLeft(s, " \t")+ if s == "" {+ return false+ }+ r, _ := utf8.DecodeRuneInString(s)+ return unicode.IsLetter(r) && unicode.IsLower(r)+}++// linesAreVerticallyAdjacent reports whether curr is roughly the next physical+// line below prev (not a distant paragraph). It is self-contained: the allowed+// gap scales with the box heights so no external median height is needed.+func linesAreVerticallyAdjacent(prev, curr pdf.TextBox) bool {+ if curr.Top <= prev.Top {+ return false+ }+ gap := curr.Top - prev.Bottom+ lineH := math.Max(prev.Bottom-prev.Top, curr.Bottom-curr.Top)+ if lineH <= 0 {+ lineH = 1+ }+ return gap <= lineH*2.5+}++// joinBrokenHyphen concatenates curr onto prev after dropping prev's trailing+// broken-hyphen marker, and expands prev's geometry to cover both lines.+func joinBrokenHyphen(prev, curr pdf.TextBox) pdf.TextBox {+ prev.Text = stripTrailingBrokenHyphen(prev.Text) + curr.Text+ prev.Top = math.Min(prev.Top, curr.Top)+ prev.Bottom = math.Max(prev.Bottom, curr.Bottom)+ prev.X0 = math.Min(prev.X0, curr.X0)+ prev.X1 = math.Max(prev.X1, curr.X1)+ return prev+}++// cleanIntraWordHyphens repairs broken-hyphen garbage inside a single box's+// text. U+FFFD / U+00AD are pure extraction noise and are deleted (with the+// optional trailing line-break space) so the word halves rejoin; a real hyphen+// at a line break ("re- covery") is rejoined the same way, while an in-word+// hyphen without a following space ("lithium-ion", "well-known") is kept.+func cleanIntraWordHyphens(s string) string {+ if !strings.ContainsRune(s, '\uFFFD') && !strings.ContainsRune(s, '\u00AD') && !strings.ContainsRune(s, '-') {+ return s+ }+ s = garbageHyphenRE.ReplaceAllString(s, "")+ s = lineBreakHyphenRE.ReplaceAllString(s, "$1$2")+ return s+}++// isBrokenHyphenRune reports whether r is a hyphen-like codepoint that a PDF+// may use to split a word across a line break.+func isBrokenHyphenRune(r rune) bool {+ switch r {+ case '\uFFFD', // replacement char (unresolved glyph)+ '\u00AD', // soft hyphen+ '-', // hyphen-minus+ '\u2010', // hyphen+ '\u2011': // non-breaking hyphen+ return true+ }+ return false+}diff --git a/internal/deepdoc/parser/pdf/layout/dehyphen_test.go b/internal/deepdoc/parser/pdf/layout/dehyphen_test.gonew file mode 100644index 0000000000..e357f0ef18--- /dev/null+++ b/internal/deepdoc/parser/pdf/layout/dehyphen_test.go@@ -0,0 +1,191 @@+package layout++import (+ "strings"+ "testing"++ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// mkHyphenBox builds a TextBox with the given text on a single line at the given+// top, with an approximate 10pt height.+func mkHyphenBox(text string, top, colID, page int) pdf.TextBox {+ return pdf.TextBox{+ Text: text,+ Top: float64(top),+ Bottom: float64(top) + 10,+ X0: 100,+ X1: 100 + float64(len(text)*5),+ ColID: colID,+ PageNumber: page,+ }+}++func TestDehyphenateFFFD(t *testing.T) {+ // "pre" + U+FFFD at line end, "cipitation" continues on next line.+ in := []pdf.TextBox{+ mkHyphenBox("pre\ufffd", 100, 1, 1),+ mkHyphenBox("cipitation", 112, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 {+ t.Fatalf("expected 1 box, got %d: %+v", len(out), out)+ }+ if out[0].Text != "precipitation" {+ t.Errorf("expected \"precipitation\", got %q", out[0].Text)+ }+}++func TestDehyphenateSoftHyphen(t *testing.T) {+ in := []pdf.TextBox{+ mkHyphenBox("sepa\u00ad", 100, 1, 1),+ mkHyphenBox("ration", 112, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "separation" {+ t.Errorf("expected rejoined \"separation\", got %+v", out)+ }+}++func TestDehyphenateASCIIHyphen(t *testing.T) {+ in := []pdf.TextBox{+ mkHyphenBox("re-", 100, 1, 1),+ mkHyphenBox("covery", 112, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "recovery" {+ t.Errorf("expected rejoined \"recovery\", got %+v", out)+ }+}++func TestDehyphenateNoMergeUppercaseContinuation(t *testing.T) {+ // Line ends with '-' but the next line starts with a capital → new+ // sentence / dash, not a hyphenated word. Must NOT merge.+ in := []pdf.TextBox{+ mkHyphenBox("word-", 100, 1, 1),+ mkHyphenBox("Word", 112, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 2 {+ t.Fatalf("expected 2 boxes (no merge), got %d: %+v", len(out), out)+ }+ if out[0].Text != "word-" || out[1].Text != "Word" {+ t.Errorf("boxes changed unexpectedly: %q / %q", out[0].Text, out[1].Text)+ }+}++func TestDehyphenateNoMergeDifferentColumn(t *testing.T) {+ in := []pdf.TextBox{+ mkHyphenBox("pre\ufffd", 100, 1, 1),+ mkHyphenBox("cipitation", 112, 2, 1), // different column+ }+ out := Dehyphenate(in)+ if len(out) != 2 {+ t.Errorf("expected 2 boxes (different columns), got %d", len(out))+ }+}++func TestDehyphenateNoMergeDifferentPage(t *testing.T) {+ in := []pdf.TextBox{+ mkHyphenBox("pre\ufffd", 100, 1, 1),+ mkHyphenBox("cipitation", 112, 1, 2), // different page+ }+ out := Dehyphenate(in)+ if len(out) != 2 {+ t.Errorf("expected 2 boxes (different pages), got %d", len(out))+ }+}++func TestDehyphenateNoMergeNonAdjacentLines(t *testing.T) {+ // Marker present and lowercase continuation, but lines are far apart.+ in := []pdf.TextBox{+ mkHyphenBox("pre\ufffd", 100, 1, 1),+ mkHyphenBox("cipitation", 300, 1, 1), // huge vertical gap+ }+ out := Dehyphenate(in)+ if len(out) != 2 {+ t.Errorf("expected 2 boxes (non-adjacent lines), got %d", len(out))+ }+}++func TestDehyphenateCleanIntraWordGarbage(t *testing.T) {+ // A replacement char embedded between two letters inside one box.+ in := []pdf.TextBox{+ mkHyphenBox("a\ufffdb", 100, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "ab" {+ t.Errorf("expected \"ab\", got %q", out[0].Text)+ }+}++func TestDehyphenatePreservesRealHyphen(t *testing.T) {+ // A literal hyphen inside a single word ("lithium-ion") must survive.+ in := []pdf.TextBox{+ mkHyphenBox("lithium-ion", 100, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "lithium-ion" {+ t.Errorf("expected \"lithium-ion\" preserved, got %q", out[0].Text)+ }+}++func TestDehyphenateWellKnownPreserved(t *testing.T) {+ // "well-known" has a real in-word hyphen with no following space.+ in := []pdf.TextBox{+ mkHyphenBox("well-known", 100, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "well-known" {+ t.Errorf("expected \"well-known\" preserved, got %q", out[0].Text)+ }+}++func TestDehyphenateMidBoxFFFDWithSpace(t *testing.T) {+ // Real-PDF shape: FFFD mid-box followed by the line-break space, then the+ // continuation word (e.g. "devel� inorganic").+ in := []pdf.TextBox{+ mkHyphenBox("devel\ufffd inorganic", 100, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "develinorganic" {+ t.Errorf("expected \"develinorganic\", got %q", out[0].Text)+ }+}++func TestDehyphenateLineBreakASCIIHyphenWithSpace(t *testing.T) {+ // "re-" at a line break followed by the line-break space and "covery".+ in := []pdf.TextBox{+ mkHyphenBox("re- covery", 100, 1, 1),+ }+ out := Dehyphenate(in)+ if len(out) != 1 || out[0].Text != "recovery" {+ t.Errorf("expected \"recovery\", got %q", out[0].Text)+ }+}++func TestDehyphenateWholePageAssembly(t *testing.T) {+ // Two independent hyphenated breaks plus an unrelated line.+ in := []pdf.TextBox{+ mkHyphenBox("pre\ufffd", 100, 1, 1),+ mkHyphenBox("cipitation", 112, 1, 1),+ mkHyphenBox("sepa\ufffd", 130, 1, 1),+ mkHyphenBox("ration", 142, 1, 1),+ mkHyphenBox("lithium-ion", 160, 1, 1),+ }+ out := Dehyphenate(in)+ var sb strings.Builder+ for _, b := range out {+ sb.WriteString(b.Text)+ sb.WriteString(" ")+ }+ got := sb.String()+ for _, want := range []string{"precipitation", "separation", "lithium-ion"} {+ if !strings.Contains(got, want) {+ t.Errorf("assembled text %q missing %q", got, want)+ }+ }+ if strings.Contains(got, "\ufffd") {+ t.Errorf("assembled text still contains U+FFFD: %q", got)+ }+}diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.goindex a5af099654..663a25c5a1 100644--- a/internal/deepdoc/parser/pdf/parser.go+++ b/internal/deepdoc/parser/pdf/parser.go@@ -592,6 +592,12 @@ func (p *Parser) buildLayout(ctx context.Context,// page-leading title isolated in its own column.boxes = lyt.FinalReadingOrderMerge(boxes)+ // Rejoin words a PDF split across a line break with a hyphen. Must run+ // after FinalReadingOrderMerge (consecutive lines are adjacent, same+ // column) and before NaiveVerticalMerge (which would otherwise glue the+ // halves with a space). See layout.Dehyphenate.+ boxes = lyt.Dehyphenate(boxes)+boxes = lyt.NaiveVerticalMerge(boxes, medianHeights, medianWidths, pageEnglish)result.Metrics.BoxesVertMerge = len(boxes)if err := ctx.Err(); err != nil {diff --git a/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.go b/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.gonew file mode 100644index 0000000000..b8ddd3b09e--- /dev/null+++ b/internal/deepdoc/parser/pdf/pdfoxide/dehyphen_pdf_test.go@@ -0,0 +1,80 @@+//go:build manual++package pdfoxide++import (+ "os"+ "strings"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/layout"+ pdf "ragflow/internal/deepdoc/parser/pdf/type"+)++// TestDehyphenateRealPDF is a regression test against the Elsevier article+// that emits U+FFFD at every line-break hyphen (pre-cipitation, sepa-ration,+// re-covery, ...). It is manual-tier and gated on RAGFLOW_TEST_PDF so it+// never runs in CI and does not require committing the 1.5 MB PDF.+//+// Run locally with:+//+// RAGFLOW_TEST_PDF=~/Downloads/1-s2.0-S0956053X25001680-main.pdf \+// bash build.sh --test-manual ./internal/deepdoc/parser/pdf/pdfoxide/ -run TestDehyphenateRealPDF -v+func TestDehyphenateRealPDF(t *testing.T) {+ path := os.Getenv("RAGFLOW_TEST_PDF")+ if path == "" {+ t.Skip("set RAGFLOW_TEST_PDF to the Elsevier PDF to run this regression test")+ }++ doc, err := Open(path)+ if err != nil {+ t.Fatalf("open %s: %v", path, err)+ }+ defer doc.Close()++ chars, err := doc.GetPageChars(0)+ if err != nil {+ t.Fatalf("get page chars: %v", err)+ }++ // Convert the extractor's local char type into the layout TextChar type.+ tchars := make([]pdf.TextChar, len(chars))+ for i, c := range chars {+ tchars[i] = pdf.TextChar{+ X0: c.X0,+ X1: c.X1,+ Top: c.Top,+ Bottom: c.Bottom,+ Text: c.Text,+ PageNumber: c.PageNumber,+ }+ }++ boxes := layout.CharsToBoxes(tchars, 1, false)+ boxes = layout.Dehyphenate(boxes)++ var sb strings.Builder+ for _, b := range boxes {+ sb.WriteString(b.Text)+ sb.WriteString("\n")+ }+ text := sb.String()++ if strings.Contains(text, "\ufffd") {+ t.Errorf("output still contains U+FFFD replacement char:\n%s", text)+ }++ // The three words the user reported must be rejoined, not broken.+ for _, want := range []string{"precipitation", "separation", "recovery"} {+ if !strings.Contains(text, want) {+ t.Errorf("expected rejoined word %q in output, got:\n%s", want, text)+ }+ }++ // And the broken forms must be gone.+ for _, bad := range []string{"pre- cipitation", "pre\u00ad", "sepa\u00ad", "re\u00ad"} {+ if strings.Contains(text, bad) {+ t.Errorf("broken hyphenation not cleaned, found %q in:\n%s", bad, text)+ }+ }+}