Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): collapse docElementText into docxElementText (#19135)

xugangqianga5a83a565302
diff --git a/internal/parser/parser/doc_element_text_test.go b/internal/parser/parser/doc_element_text_test.go
new file mode 100644
index 0000000000..3b8834cac3
--- /dev/null
+++ b/internal/parser/parser/doc_element_text_test.go
@@ -0,0 +1,39 @@
+package parser
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+// cellWithText builds a table cell whose only content is a single paragraph
+// run carrying s, so joinCellText renders it as s.
+func cellWithText(s string) docxIRCell {
+ return docxIRCell{
+ Content: []docxIRElement{{
+ Type: "paragraph",
+ Content: json.RawMessage(`[{"type":"text","text":"` + s + `"}]`),
+ }},
+ }
+}
+
+// TestDocxElementTextCellSeparator locks the contract introduced when
+// docElementText was collapsed into docxElementText: the cellSep argument is
+// the within-row separator. docx/pptx pass "\n" (each cell on its own line);
+// the .doc flatten path passes " | " so recovered table text stays readable.
+func TestDocxElementTextCellSeparator(t *testing.T) {
+ table := docxIRElement{
+ Type: "table",
+ Rows: []docxIRRow{
+ {Cells: []docxIRCell{cellWithText("a"), cellWithText("b")}},
+ {Cells: []docxIRCell{cellWithText("c"), cellWithText("d")}},
+ },
+ }
+
+ if got := docxElementText(table, "\n"); got != "a\nb\nc\nd" {
+ t.Errorf("docx cellSep=\"\\n\": got %q, want %q", got, "a\nb\nc\nd")
+ }
+
+ if got := docxElementText(table, " | "); got != "a | b\nc | d" {
+ t.Errorf("doc cellSep=\" | \": got %q, want %q", got, "a | b\nc | d")
+ }
+}
diff --git a/internal/parser/parser/doc_flatten.go b/internal/parser/parser/doc_flatten.go
index c5091a4d17..054f5cbc18 100644
--- a/internal/parser/parser/doc_flatten.go
+++ b/internal/parser/parser/doc_flatten.go
@@ -54,43 +54,10 @@ func flattenDocIR(irJSON string) string {
parts = append(parts, t)
}
for _, el := range sec.Elements {
- if t := docElementText(el); t != "" {
+ if t := docxElementText(el, " | "); t != "" {
parts = append(parts, t)
}
}
}
return strings.Join(parts, "\n")
}
-
-// docElementText returns the plain-text rendering of a single IR element for
-// the .doc flatten path. It mirrors docxElementText but separates table cells
-// with " | " (docxElementText concatenates them) so flattened table text
-// stays readable. Returns "" for image and unknown types.
-func docElementText(el docxIRElement) string {
- switch el.Type {
- case "paragraph", "heading":
- return joinDOCXIRRuns(el.contentRuns())
- case "table":
- var rows []string
- for _, row := range el.Rows {
- var cells []string
- for _, cell := range row.Cells {
- cells = append(cells, joinCellText(cell))
- }
- rows = append(rows, strings.Join(cells, " | "))
- }
- return strings.Join(rows, "\n")
- case "list":
- var lines []string
- for _, item := range el.Items {
- if t := extractTextFromListItem(item); t != "" {
- lines = append(lines, t)
- }
- }
- return strings.Join(lines, "\n")
- case "text_box":
- return extractTextFromBlockElements(el.contentBlocks())
- default:
- return ""
- }
-}
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index 27d140cd6c..c5a84e2291 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -209,22 +209,29 @@ func docxIRTableToHTML(el docxIRElement) string {
// A bare "text" block is not part of the current IR schema, but is
// passed through so its payload is never silently dropped. Returns ""
// for image and unknown types.
-func docxElementText(el docxIRElement) string {
+//
+// For table elements, cellSep joins the cells within a single row. Pass
+// "\n" to put each cell on its own line (the pptx/docx flatten behavior,
+// equivalent to the previous flat-all-cells rendering), or " | " to keep a
+// row's cells on one line (the legacy .doc viewport).
+func docxElementText(el docxIRElement, cellSep string) string {
switch el.Type {
case "paragraph", "heading":
return joinDOCXIRRuns(el.contentRuns())
case "text":
return el.Text
case "table":
- var lines []string
+ var rows []string
for _, row := range el.Rows {
+ var cells []string
for _, cell := range row.Cells {
if t := joinCellText(cell); t != "" {
- lines = append(lines, t)
+ cells = append(cells, t)
}
}
+ rows = append(rows, strings.Join(cells, cellSep))
}
- return strings.Join(lines, "\n")
+ return strings.Join(rows, "\n")
case "list":
var lines []string
for _, item := range el.Items {
@@ -339,7 +346,7 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
flat = append(flat, flatBlock{image: b64})
continue
}
- text := docxElementText(el)
+ text := docxElementText(el, "\n")
flat = append(flat, flatBlock{text: text})
}
}
diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.go
index cfe412646e..a440796ca6 100644
--- a/internal/parser/parser/pptx_ir.go
+++ b/internal/parser/parser/pptx_ir.go
@@ -45,10 +45,11 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {
for _, el := range sec.Elements {
// Each element's full text (paragraphs, table cells, list
// items, etc.) becomes a single value; the shared IR walker
- // already inserts newlines between rows/items, so internal
- // newlines are preserved as-is. Only the element-level split
- // is collapsed (one element → one value or none if empty).
- if text := strings.TrimSpace(docxElementText(el)); text != "" {
+ // already inserts newlines between rows/items (cellSep "\n"),
+ // so internal newlines are preserved as-is. Only the
+ // element-level split is collapsed (one element → one value or
+ // none if empty).
+ if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {
lines = append(lines, text)
}
}