RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): collapse docElementText into docxElementText (#19135)
xugangqianga5a83a565302
diff --git a/internal/parser/parser/doc_element_text_test.go b/internal/parser/parser/doc_element_text_test.gonew file mode 100644index 0000000000..3b8834cac3--- /dev/null+++ b/internal/parser/parser/doc_element_text_test.go@@ -0,0 +1,39 @@+package parser++import (+ "encoding/json"+ "testing"+)++// cellWithText builds a table cell whose only content is a single paragraph+// run carrying s, so joinCellText renders it as s.+func cellWithText(s string) docxIRCell {+ return docxIRCell{+ Content: []docxIRElement{{+ Type: "paragraph",+ Content: json.RawMessage(`[{"type":"text","text":"` + s + `"}]`),+ }},+ }+}++// TestDocxElementTextCellSeparator locks the contract introduced when+// docElementText was collapsed into docxElementText: the cellSep argument is+// the within-row separator. docx/pptx pass "\n" (each cell on its own line);+// the .doc flatten path passes " | " so recovered table text stays readable.+func TestDocxElementTextCellSeparator(t *testing.T) {+ table := docxIRElement{+ Type: "table",+ Rows: []docxIRRow{+ {Cells: []docxIRCell{cellWithText("a"), cellWithText("b")}},+ {Cells: []docxIRCell{cellWithText("c"), cellWithText("d")}},+ },+ }++ if got := docxElementText(table, "\n"); got != "a\nb\nc\nd" {+ t.Errorf("docx cellSep=\"\\n\": got %q, want %q", got, "a\nb\nc\nd")+ }++ if got := docxElementText(table, " | "); got != "a | b\nc | d" {+ t.Errorf("doc cellSep=\" | \": got %q, want %q", got, "a | b\nc | d")+ }+}diff --git a/internal/parser/parser/doc_flatten.go b/internal/parser/parser/doc_flatten.goindex c5091a4d17..054f5cbc18 100644--- a/internal/parser/parser/doc_flatten.go+++ b/internal/parser/parser/doc_flatten.go@@ -54,43 +54,10 @@ func flattenDocIR(irJSON string) string {parts = append(parts, t)}for _, el := range sec.Elements {- if t := docElementText(el); t != "" {+ if t := docxElementText(el, " | "); t != "" {parts = append(parts, t)}}}return strings.Join(parts, "\n")}--// docElementText returns the plain-text rendering of a single IR element for-// the .doc flatten path. It mirrors docxElementText but separates table cells-// with " | " (docxElementText concatenates them) so flattened table text-// stays readable. Returns "" for image and unknown types.-func docElementText(el docxIRElement) string {- switch el.Type {- case "paragraph", "heading":- return joinDOCXIRRuns(el.contentRuns())- case "table":- var rows []string- for _, row := range el.Rows {- var cells []string- for _, cell := range row.Cells {- cells = append(cells, joinCellText(cell))- }- rows = append(rows, strings.Join(cells, " | "))- }- return strings.Join(rows, "\n")- case "list":- var lines []string- for _, item := range el.Items {- if t := extractTextFromListItem(item); t != "" {- lines = append(lines, t)- }- }- return strings.Join(lines, "\n")- case "text_box":- return extractTextFromBlockElements(el.contentBlocks())- default:- return ""- }-}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 27d140cd6c..c5a84e2291 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -209,22 +209,29 @@ func docxIRTableToHTML(el docxIRElement) string {// A bare "text" block is not part of the current IR schema, but is// passed through so its payload is never silently dropped. Returns ""// for image and unknown types.-func docxElementText(el docxIRElement) string {+//+// For table elements, cellSep joins the cells within a single row. Pass+// "\n" to put each cell on its own line (the pptx/docx flatten behavior,+// equivalent to the previous flat-all-cells rendering), or " | " to keep a+// row's cells on one line (the legacy .doc viewport).+func docxElementText(el docxIRElement, cellSep string) string {switch el.Type {case "paragraph", "heading":return joinDOCXIRRuns(el.contentRuns())case "text":return el.Textcase "table":- var lines []string+ var rows []stringfor _, row := range el.Rows {+ var cells []stringfor _, cell := range row.Cells {if t := joinCellText(cell); t != "" {- lines = append(lines, t)+ cells = append(cells, t)}}+ rows = append(rows, strings.Join(cells, cellSep))}- return strings.Join(lines, "\n")+ return strings.Join(rows, "\n")case "list":var lines []stringfor _, item := range el.Items {@@ -339,7 +346,7 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {flat = append(flat, flatBlock{image: b64})continue}- text := docxElementText(el)+ text := docxElementText(el, "\n")flat = append(flat, flatBlock{text: text})}}diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex cfe412646e..a440796ca6 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -45,10 +45,11 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {for _, el := range sec.Elements {// Each element's full text (paragraphs, table cells, list// items, etc.) becomes a single value; the shared IR walker- // already inserts newlines between rows/items, so internal- // newlines are preserved as-is. Only the element-level split- // is collapsed (one element → one value or none if empty).- if text := strings.TrimSpace(docxElementText(el)); text != "" {+ // already inserts newlines between rows/items (cellSep "\n"),+ // so internal newlines are preserved as-is. Only the+ // element-level split is collapsed (one element → one value or+ // none if empty).+ if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {lines = append(lines, text)}}