RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): recover text_box and nested table/list text in office IR (#19319)
jay77721e53e5de0cb93
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex c5a84e2291..71286bea47 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -73,7 +73,11 @@ func (e docxIRElement) contentRuns() []docxIRRun {return runs}-// contentBlocks decodes Content as block-level elements (text_box type).+// contentBlocks decodes Content as block-level elements (text_box type,+// list-item content, or table-cell content). Blocks may be plain paragraphs+// or compound elements (table, list, nested text_box); callers flatten each+// block via docxElementText so non-paragraph blocks are not silently+// dropped.func (e docxIRElement) contentBlocks() []docxIRElement {var blocks []docxIRElementif len(e.Content) > 0 {@@ -84,7 +88,7 @@ func (e docxIRElement) contentBlocks() []docxIRElement {// docxIRListItem represents one item in an ordered/unordered list.type docxIRListItem struct {- Content []docxIRElement `json:"content"` // block-level content (typically a single Paragraph)+ Content []docxIRElement `json:"content"` // block-level content (typically a single Paragraph; may also hold table/list/text_box)Nested *docxIRList `json:"nested,omitempty"` // optional nested sub-list; null/absent when none}@@ -106,7 +110,7 @@ type docxIRRow struct {}type docxIRCell struct {- Content []docxIRElement `json:"content"` // nested paragraphs inside table cell+ Content []docxIRElement `json:"content"` // block-level content (typically paragraphs; may also hold list/table)}// joinDOCXIRRuns concatenates inline runs into plain text. Hard line@@ -126,18 +130,16 @@ func joinDOCXIRRuns(runs []docxIRRun) string {}// extractTextFromListItem extracts the plain text content from a list item.-// Each list item contains block-level elements (typically a Paragraph),-// whose text runs are concatenated. Nested sub-lists (multi-level-// bullets/numbered items) are decoded and recursed so their text is not-// silently dropped. Mirrors office_oxide ir::ListItem { content, nested }.+// Each block in the item is flattened via docxElementText so non-paragraph+// blocks (e.g. a table nested in a list item) are not silently dropped.+// Nested sub-lists (multi-level bullets/numbered items) are decoded and+// recursed so their text is not silently dropped.+// Mirrors office_oxide ir::ListItem { content, nested }.func extractTextFromListItem(item docxIRListItem) string {var parts []stringfor _, el := range item.Content {- if el.Type == "paragraph" || el.Type == "heading" {- t := joinDOCXIRRuns(el.contentRuns())- if t != "" {- parts = append(parts, t)- }+ if t := strings.TrimSpace(docxElementText(el, "\n")); t != "" {+ parts = append(parts, t)}}if item.Nested != nil {@@ -154,16 +156,15 @@ func extractTextFromListItem(item docxIRListItem) string {}// extractTextFromBlockElements extracts text from a slice of block-level-// elements (paragraphs/headings), used by text_box and other compound-// element types.+// elements (e.g. the content of a text_box). Each block is flattened via+// docxElementText so non-paragraph blocks (e.g. a table wrapped in a+// text_box, as office_oxide emits for grouped slide shapes) are not+// silently dropped.func extractTextFromBlockElements(blocks []docxIRElement) string {var parts []stringfor _, el := range blocks {- if el.Type == "paragraph" || el.Type == "heading" {- t := joinDOCXIRRuns(el.contentRuns())- if t != "" {- parts = append(parts, t)- }+ if t := strings.TrimSpace(docxElementText(el, "\n")); t != "" {+ parts = append(parts, t)}}if len(parts) == 0 {@@ -172,12 +173,13 @@ func extractTextFromBlockElements(blocks []docxIRElement) string {return strings.TrimSpace(strings.Join(parts, "\n"))}-// joinCellText concatenates all paragraph texts inside a table cell,-// joined by newlines.+// joinCellText concatenates all block texts inside a table cell, joined by+// newlines. Each block is flattened via docxElementText so non-paragraph+// blocks (e.g. a nested list) are not silently dropped.func joinCellText(cell docxIRCell) string {var parts []stringfor _, el := range cell.Content {- if text := joinDOCXIRRuns(el.contentRuns()); text != "" {+ if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {parts = append(parts, text)}}diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 416f6899b9..48b7b9196e 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -244,6 +244,83 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {}}+// TestExtractTextFromListItem_NestedTable verifies that a table nested+// inside a list item's content is flattened, not silently dropped. The old+// path only recognized paragraph/heading blocks in item.Content.+func TestExtractTextFromListItem_NestedTable(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"list","items":[+ {"content":[+ {"type":"paragraph","content":[{"type":"text","text":"before"}]},+ {"type":"table","rows":[+ {"cells":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"c1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"c2"}]}]}+ ]}+ ]}+ ]}+ ]}+ ]}]}`+ sections := buildDOCXJSONSections(irJSON)+ if len(sections) != 1 {+ t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)+ }+ if got := sections[0]["text"]; got != "before\nc1\nc2" {+ t.Errorf("list item with nested table text = %q, want %q", got, "before\nc1\nc2")+ }+}++// TestJoinCellText_NestedList verifies that a list nested inside a table+// cell is flattened, not silently dropped. The old path only recognized+// paragraph runs in cell.Content.+func TestJoinCellText_NestedList(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"table","rows":[+ {"cells":[+ {"content":[+ {"type":"paragraph","content":[{"type":"text","text":"head"}]},+ {"type":"list","items":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li2"}]}]}+ ]}+ ]}+ ]}+ ]}+ ]}]}`+ var ir docxIRDocument+ if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {+ t.Fatalf("unmarshal IR: %v", err)+ }+ cell := ir.Sections[0].Elements[0].Rows[0].Cells[0]+ if got := joinCellText(cell); got != "head\nli1\nli2" {+ t.Errorf("joinCellText(nested list) = %q, want %q", got, "head\nli1\nli2")+ }+}++// TestBuildDOCXJSONSections_TextBoxTable verifies end-to-end that a table+// wrapped in a text_box (the shape office_oxide emits for grouped slide+// shapes, and a legal DOCX text-box shape) survives into the section+// output. Mirrors the PPTX bwbd.pptx regression at the DOCX layer.+func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"text_box","content":[+ {"type":"table","rows":[+ {"cells":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"Q1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"A1"}]}]}+ ]}+ ]}+ ]}+ ]}]}`+ sections := buildDOCXJSONSections(irJSON)+ if len(sections) != 1 {+ t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)+ }+ if got := sections[0]["text"]; got != "Q1\nA1" {+ t.Errorf("text_box table text = %q, want %q", got, "Q1\nA1")+ }+}+// TestBuildDOCXJSONSections_NestedList verifies end-to-end that a// multi-level list survives into the section output with all levels.func TestBuildDOCXJSONSections_NestedList(t *testing.T) {diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex a440796ca6..e0507e53d0 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -62,6 +62,19 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {return items, nil}+// itemsAllEmpty reports whether every item carries no extractable text.+// A deck whose IR sections all flatten to "" (e.g. text the IR walker does+// not yet cover) must fall back to PlainText instead of emitting empty+// chunks that the Tokenizer later filters, silently yielding 0 chunks.+func itemsAllEmpty(items []map[string]any) bool {+ for _, it := range items {+ if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {+ return false+ }+ }+ return true+}+// itemsFromPlainText wraps whole-document plain text as a single JSON// item. It salvages a still-readable deck when the structured IR cannot// be used (IR serialization failure or a sectionless IR); a documentdiff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.goindex 0444fa2569..7e5d708df4 100644--- a/internal/parser/parser/pptx_ir_test.go+++ b/internal/parser/parser/pptx_ir_test.go@@ -108,6 +108,61 @@ func TestBuildPPTXJSONSections(t *testing.T) {irJSON: `{"sections":[{"elements":[{"type":"image","data":"aGVsbG8="}]}]}`,wantTexts: []string{""},},+ {+ // bwbd.pptx regression: office_oxide wraps grouped slide shapes+ // (graphicFrame tables) in a text_box, so the slide's text_box+ // content is a table, not paragraphs. Previously flattened to "".+ name: "text_box wrapping table keeps text",+ irJSON: `{"sections":[{"elements":[{"type":"text_box","content":[+ {"type":"table","rows":[+ {"cells":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"Q1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"A1"}]}]}+ ]}+ ]}+ ]}]}]}`,+ wantTexts: []string{"Q1\nA1"},+ },+ {+ // Same flattening path with a list inside the text_box: the+ // old paragraph-only walk dropped the whole block.+ name: "text_box wrapping list keeps text",+ irJSON: `{"sections":[{"elements":[{"type":"text_box","content":[+ {"type":"list","items":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li2"}]}]}+ ]}+ ]}]}]}`,+ wantTexts: []string{"li1\nli2"},+ },+ {+ // Nested text boxes recurse through the same walker.+ name: "nested text_box keeps text",+ irJSON: `{"sections":[{"elements":[{"type":"text_box","content":[+ {"type":"paragraph","content":[{"type":"text","text":"outer"}]},+ {"type":"text_box","content":[+ {"type":"paragraph","content":[{"type":"text","text":"inner"}]}+ ]}+ ]}]}]}`,+ wantTexts: []string{"outer\ninner"},+ },+ {+ // Table cell holding a list: joinCellText must recurse instead+ // of reading paragraph runs only.+ name: "table cell wrapping list keeps text",+ irJSON: `{"sections":[{"elements":[{"type":"table","rows":[+ {"cells":[+ {"content":[+ {"type":"paragraph","content":[{"type":"text","text":"head"}]},+ {"type":"list","items":[+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li1"}]}]},+ {"content":[{"type":"paragraph","content":[{"type":"text","text":"li2"}]}]}+ ]}+ ]}+ ]}+ ]}]}]}`,+ wantTexts: []string{"head\nli1\nli2"},+ },{name: "invalid JSON",irJSON: "{not json",@@ -149,6 +204,36 @@ func TestBuildPPTXJSONSections(t *testing.T) {}}+// TestItemsAllEmpty pins the all-empty fallback gate: empty and whitespace+// items trigger the PlainText salvage, any non-empty item does not.+func TestItemsAllEmpty(t *testing.T) {+ tests := []struct {+ name string+ items []map[string]any+ want bool+ }{+ {name: "nil", items: nil, want: true},+ {name: "empty", items: []map[string]any{}, want: true},+ {+ name: "all empty",+ items: []map[string]any{{"text": ""}, {"text": " \n "}},+ want: true,+ },+ {+ name: "one non-empty",+ items: []map[string]any{{"text": ""}, {"text": "hello"}},+ want: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := itemsAllEmpty(tt.items); got != tt.want {+ t.Errorf("itemsAllEmpty = %v, want %v", got, tt.want)+ }+ })+ }+}+// TestItemsFromPlainText pins the whole-document plain-text salvage used// when the structured IR is unusable: non-empty text becomes a single// trimmed item; blank text yields an empty, non-nil slice.diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex 149567a29d..059cd2f97b 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -168,10 +168,13 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, dataif err != nil {return ParseResult{Err: err}}- if len(items) == 0 {- // A deck whose IR carries no sections at all: keep the- // whole-document text as a single item so a still-readable- // file yields one chunk instead of none.+ if len(items) == 0 || itemsAllEmpty(items) {+ // A deck whose IR carries no sections at all, or whose sections+ // all flatten to empty text (text the IR walker does not cover):+ // keep the whole-document text as a single item so a+ // still-readable file yields content instead of none. Without+ // the all-empty check, empty items flow to the Tokenizer, which+ // filters them as empty text, silently yielding 0 chunks.text, perr := doc.PlainText()if perr != nil {return ParseResult{Err: fmt.Errorf("presentation plain-text: %w", perr)}