RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: fall back to the real office container format when extension is wrong (#18843)
jay77721e9d5dff6281a
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex e6c0972c53..54833ccef4 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -26,6 +26,12 @@ import (officeOxide "github.com/yfedoseev/office_oxide/go")+// docxOpenFromBytes is a test seam that mirrors officeOxide.OpenFromBytes.+// Overridden in tests to capture the effective container format.+// Not safe for concurrent use with t.Parallel() — tests must save/restore+// it sequentially (see office_detect_cgo_test.go).+var docxOpenFromBytes = officeOxide.OpenFromBytes+// DOCXParser is the cgo-backed DOCX parser. It is the only DOCX// entrypoint that depends on office_oxide; the IR data model and// postprocessing live in cgo-free files (docx_ir.go, docx_postprocess.go)@@ -67,8 +73,28 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {//// JSON path mirrors python parser.py:_docx() output_format == "json".// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().+//+// File["format"] in the returned ParseResult reflects the effective+// container format after magic-byte sniffing (e.g. "doc" for an OLE+// payload uploaded as .docx), not merely the file extension. See+// office_detect_cgo.go:officeContainer for the detection contract.func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- doc, err := officeOxide.OpenFromBytes(data, "docx")+ // office_oxide's OpenFromBytes takes the container format as given+ // and does no magic-byte detection, so a legacy .doc uploaded with a+ // .docx extension would be parsed as ZIP/OOXML and fail. Sniff the+ // real container and pass the matching format so mislabeled files+ // still parse (mirrors the magic-byte correction office_oxide::Open+ // performs on file paths).+ format := "docx"+ if officeContainer(data) == "ole" {+ format = "doc"+ }+ // Note: when format == "doc" (OLE fallback), header/footer and TOC+ // post-processing below degrades gracefully: extractDOCXHeaderFooterTexts+ // opens the payload as ZIP and fails closed, so RemoveHeaderFooter becomes+ // a no-op for legacy OLE inputs. This is acceptable — legacy DOC has no+ // OOXML header/footer parts to strip.+ doc, err := docxOpenFromBytes(data, format)if err != nil {return ParseResult{Err: fmt.Errorf("docx open: %w", err)}}@@ -76,7 +102,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, datafileMeta := map[string]any{"name": filename,- "format": "docx",+ "format": format,}// Extract IR JSON for section building (JSON path) anddiff --git a/internal/parser/parser/office_detect_cgo.go b/internal/parser/parser/office_detect_cgo.gonew file mode 100644index 0000000000..7e8a3cb47c--- /dev/null+++ b/internal/parser/parser/office_detect_cgo.go@@ -0,0 +1,47 @@+//go:build cgo++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++// officeContainer reports the on-disk container format of an Office+// document by sniffing its leading magic bytes. It returns "ooxml" for+// the ZIP-based OOXML family (docx/xlsx/pptx), "ole" for the legacy+// binary OLE family (doc/xls/ppt), or "" when the container cannot be+// determined from the first bytes. The return values are internal+// markers — callers map them to the concrete office_oxide format strings+// ("docx"→"doc", "pptx"→"ppt"); this avoids coupling the detector to a+// single family and keeps it reusable for xlsx/xls if needed later.+//+// The office_oxide backend's OpenFromBytes requires the caller to name+// the container format and does no magic-byte detection of its own, so+// a legacy file renamed to an OOXML extension (e.g. a .doc uploaded as+// .docx) is otherwise parsed as ZIP and fails with OFFICE_ERR_PARSE.+// Sniffing here lets the cgo parsers pass the real container format.+func officeContainer(data []byte) string {+ if len(data) >= 4 && data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04 {+ return "ooxml"+ }+ // OLE Compound Document signature: D0 CF 11 E0 A1 B1 1A E1. Require+ // the full eight bytes to avoid misclassifying truncated input+ // (a four-byte prefix could also be garbage).+ if len(data) >= 8 && data[0] == 0xD0 && data[1] == 0xCF && data[2] == 0x11 && data[3] == 0xE0 &&+ data[4] == 0xA1 && data[5] == 0xB1 && data[6] == 0x1A && data[7] == 0xE1 {+ return "ole"+ }+ return ""+}diff --git a/internal/parser/parser/office_detect_cgo_test.go b/internal/parser/parser/office_detect_cgo_test.gonew file mode 100644index 0000000000..73e0b0e810--- /dev/null+++ b/internal/parser/parser/office_detect_cgo_test.go@@ -0,0 +1,224 @@+//go:build cgo++package parser++import (+ "bytes"+ "fmt"+ "testing"++ officeOxide "github.com/yfedoseev/office_oxide/go"+)++// TestOfficeContainer verifies the magic-byte sniffing that lets the+// office_oxide parsers recover from a mislabeled extension (e.g. a+// legacy .doc uploaded as .docx). office_oxide::OpenFromBytes takes the+// container format on trust and does no detection of its own.+func TestOfficeContainer(t *testing.T) {+ tests := []struct {+ name string+ data []byte+ want string+ }{+ {"ooxml", []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}, "ooxml"},+ {"ole", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}, "ole"},+ {"ole_with_trailing", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0xFF}, "ole"},+ {"ole_short_header", []byte{0xD0, 0xCF, 0x11, 0xE0}, ""},+ {"ole_truncated_7", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A}, ""},+ {"empty", []byte{}, ""},+ {"too_short", []byte{0x50, 0x4B}, ""},+ {"ooxml_truncated_3", []byte{0x50, 0x4B, 0x03}, ""},+ {"garbage", []byte{0x00, 0x11, 0x22, 0x33}, ""},+ {"ole_prefix_mismatch", []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0x00}, ""},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := officeContainer(tt.data); got != tt.want {+ t.Errorf("officeContainer(% x) = %q, want %q", tt.data, got, tt.want)+ }+ })+ }+}++// TestDOCXParser_FallsBackToDOCForOLEHeader ensures a legacy OLE .doc+// payload routed to the DOCXParser (the .doc-as-.docx case) is opened+// with the "doc" container format rather than "docx". A minimal OLE+// header is not a valid document, so office_oxide still errors — but+// the error must come from the DOC path, proving the fallback fired.+// The test uses a seam to capture the effective format without needing+// a full valid OLE document fixture.+func TestDOCXParser_FallsBackToDOCForOLEHeader(t *testing.T) {+ ctx := t.Context()+ p := NewDOCXParser()++ orig := docxOpenFromBytes+ defer func() { docxOpenFromBytes = orig }()++ var gotFormat string+ docxOpenFromBytes = func(data []byte, format string) (*officeOxide.Document, error) {+ gotFormat = format+ return nil, fmt.Errorf("stub doc open: %s", format)+ }++ data := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}+ res := p.ParseWithResult(ctx, "renamed.docx", data)+ if res.Err == nil {+ t.Fatal("expected parse error for a non-document OLE header, got nil")+ }+ if gotFormat != "doc" {+ t.Fatalf("docxOpenFromBytes format = %q, want %q (OLE fallback not triggered)", gotFormat, "doc")+ }+}++// TestDOCXParser_TruncatedOLENotFallback ensures a truncated OLE prefix+// (only 4 bytes) does not trigger the DOC fallback — it must stay "docx"+// so truncated garbage is not misclassified as OLE.+func TestDOCXParser_TruncatedOLENotFallback(t *testing.T) {+ ctx := t.Context()+ p := NewDOCXParser()++ orig := docxOpenFromBytes+ defer func() { docxOpenFromBytes = orig }()++ var gotFormat string+ docxOpenFromBytes = func(data []byte, format string) (*officeOxide.Document, error) {+ gotFormat = format+ return nil, fmt.Errorf("stub")+ }++ data := []byte{0xD0, 0xCF, 0x11, 0xE0}+ res := p.ParseWithResult(ctx, "truncated.docx", data)+ if res.Err == nil {+ t.Fatal("expected error, got nil")+ }+ if gotFormat != "docx" {+ t.Fatalf("format = %q, want %q for truncated OLE (should not fallback)", gotFormat, "docx")+ }+}++// TestDOCXParser_OOXMLStillDOCX is the regression guard: a real OOXML+// (ZIP) payload must still take the "docx" path and parse successfully.+func TestDOCXParser_OOXMLStillDOCX(t *testing.T) {+ ctx := t.Context()+ p := NewDOCXParser()+ data := minimalDOCX(t, "round-trip OOXML")+ res := p.ParseWithResult(ctx, "real.docx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if res.OutputFormat != "markdown" {+ t.Fatalf("OutputFormat = %q, want %q", res.OutputFormat, "markdown")+ }+ if got := res.File["format"]; got != "docx" {+ t.Fatalf("File[format] = %v, want %q", got, "docx")+ }+ if !bytes.Contains([]byte(res.Markdown), []byte("round-trip OOXML")) {+ t.Fatalf("Markdown = %q, want it to contain the source text", res.Markdown)+ }+}++// TestPPTXParser_FallsBackToPPTForOLEHeader ensures a legacy OLE .ppt+// payload routed to PPTXParser is opened with "ppt" rather than "pptx".+func TestPPTXParser_FallsBackToPPTForOLEHeader(t *testing.T) {+ ctx := t.Context()+ p := NewPPTXParser()++ orig := pptxOpenFromBytes+ defer func() { pptxOpenFromBytes = orig }()++ var gotFormat string+ pptxOpenFromBytes = func(data []byte, format string) (*officeOxide.Document, error) {+ gotFormat = format+ return nil, fmt.Errorf("stub ppt open: %s", format)+ }++ data := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}+ res := p.ParseWithResult(ctx, "renamed.pptx", data)+ if res.Err == nil {+ t.Fatal("expected parse error for OLE header, got nil")+ }+ if gotFormat != "ppt" {+ t.Fatalf("pptxOpenFromBytes format = %q, want %q", gotFormat, "ppt")+ }+ if p.format != "pptx" {+ t.Fatalf("PPTXParser.format mutated to %q, want %q (must use local effFormat)", p.format, "pptx")+ }+}++// TestPPTParser_FallsBackToPPTXForOOXMLHeader ensures an OOXML .pptx+// payload uploaded as .ppt ( PPTParser with p.format == "ppt") is+// correctly routed to "pptx" via the bidirectional container sniffing.+func TestPPTParser_FallsBackToPPTXForOOXMLHeader(t *testing.T) {+ ctx := t.Context()+ p := NewPPTParser() // underlying PPTXParser{format:"ppt"}++ orig := pptxOpenFromBytes+ defer func() { pptxOpenFromBytes = orig }()++ var gotFormat string+ pptxOpenFromBytes = func(data []byte, format string) (*officeOxide.Document, error) {+ gotFormat = format+ return nil, fmt.Errorf("stub ppt open: %s", format)+ }++ // Minimal OOXML ZIP local file header.+ data := []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}+ res := p.ParseWithResult(ctx, "renamed.ppt", data)+ if res.Err == nil {+ t.Fatal("expected parse error for OOXML header stub, got nil")+ }+ if gotFormat != "pptx" {+ t.Fatalf("pptxOpenFromBytes format = %q, want %q (OOXML fallback ppt->pptx)", gotFormat, "pptx")+ }+ // Underlying PPTXParser must not be mutated permanently.+ if p.pptx.format != "ppt" {+ t.Fatalf("PPTParser.pptx.format mutated to %q, want %q (must use local effFormat)", p.pptx.format, "ppt")+ }+}++// TestPPTXParser_NoStatePollution verifies that after an OLE file mutates+// the effective format for that call, a subsequent OOXML file still uses+// "pptx" — i.e., the parser instance is not permanently polluted.+func TestPPTXParser_NoStatePollution(t *testing.T) {+ ctx := t.Context()+ p := NewPPTXParser()++ orig := pptxOpenFromBytes+ defer func() { pptxOpenFromBytes = orig }()++ calls := []string{}+ pptxOpenFromBytes = func(data []byte, format string) (*officeOxide.Document, error) {+ calls = append(calls, format)+ return nil, fmt.Errorf("stub")+ }++ oleData := []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}+ _ = p.ParseWithResult(ctx, "a.pptx", oleData)+ ooxmlData := []byte{0x50, 0x4B, 0x03, 0x04, 0x00, 0x00}+ _ = p.ParseWithResult(ctx, "b.pptx", ooxmlData)++ if len(calls) != 2 {+ t.Fatalf("calls = %v, want 2", calls)+ }+ if calls[0] != "ppt" {+ t.Errorf("first call format = %q, want %q", calls[0], "ppt")+ }+ if calls[1] != "pptx" {+ t.Errorf("second call format = %q, want %q (state polluted)", calls[1], "pptx")+ }+}++// TestPPTXParser_OOXMLStillPPTX validates the effective format and File+// metadata for a real OOXML presentation.+func TestPPTXParser_OOXMLStillPPTX(t *testing.T) {+ ctx := t.Context()+ p := NewPPTXParser()+ data := buildPPTX(t, "pptx round-trip")+ res := p.ParseWithResult(ctx, "real.pptx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got := res.File["format"]; got != "pptx" {+ t.Fatalf("File[format] = %v, want %q", got, "pptx")+ }+}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex c67c70fe7c..339e976d52 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -55,6 +55,15 @@ type ParseResult struct {// descriptor, augmented with format-specific keys (e.g.// `outline` on the PDF path, `page_count` for paginated// formats). Nil when the parser did not enrich.+ //+ // For the office family (docx/doc, pptx/ppt), File["format"]+ // reflects the real container format detected via magic-byte+ // sniffing (officeContainer), not just the file extension.+ // E.g. a legacy OLE .doc uploaded as .docx yields+ // File["format"] == "doc" so downstream consumers see the+ // effective format. This is an intentional contract change+ // from the pre-fallback behaviour where File["format"] always+ // echoed the requested extension.File map[string]any// JSON is the structured payload when OutputFormat == "json".diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.goindex 9cab23d9dd..2a1d3dad16 100644--- a/internal/parser/parser/ppt_parser_cgo_test.go+++ b/internal/parser/parser/ppt_parser_cgo_test.go@@ -51,21 +51,20 @@ func TestPPTXParser_ParseWithResult_CGO(t *testing.T) {}// TestPPTParser_ParseWithResult_CGO verifies that PPTParser-// delegates correctly to PPTXParser{format:"ppt"} and produces-// output with File["format"] = "ppt".+// delegates correctly to PPTXParser{format:"ppt"} and, via the+// bidirectional magic-byte fallback, correctly opens OOXML content+// through the PPTX path. A .pptx payload uploaded as .ppt should+// succeed and report File["format"] == "pptx" (real container).func TestPPTParser_ParseWithResult_CGO(t *testing.T) {ctx := t.Context()p := NewPPTParser()- // Use PPTX content — office_oxide may reject it with format="ppt"- // hint (expects OLE binary). When it does, skip gracefully; when- // it succeeds, verify the metadata contract.data := buildPPTX(t, "Hello")res := p.ParseWithResult(ctx, "test.ppt", data)if res.Err != nil {- t.Skip("PPTParser with PPTX data (expected maybe to fail):", res.Err)+ t.Fatalf("ParseWithResult: %v", res.Err)}- if got := res.File["format"]; got != "ppt" {- t.Errorf("File[format] = %v, want %q", got, "ppt")+ if got := res.File["format"]; got != "pptx" {+ t.Errorf("File[format] = %v, want %q (OOXML fallback ppt->pptx)", got, "pptx")}}diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex 9665f814b8..7feaa60a70 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -26,6 +26,11 @@ import (officeOxide "github.com/yfedoseev/office_oxide/go")+// pptxOpenFromBytes is a test seam mirroring officeOxide.OpenFromBytes.+// Not safe for concurrent use with t.Parallel() — tests must save/restore+// it sequentially.+var pptxOpenFromBytes = officeOxide.OpenFromBytes+// PPTXParser parses both .pptx (OOXML) and .ppt (OLE binary)// files via the office_oxide backend. The format field controls// the container format passed to OpenFromBytes — "pptx" for@@ -95,6 +100,12 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datamethod := strings.ToLower(strings.TrimSpace(p.ParseMethod))switch method {case "tcadp":+ // TCADP file_type is intentionally derived from the parser's+ // configured family (p.format == "pptx" → "PPTX", "ppt" → "PPT"),+ // not from magic-byte sniffing. The pipeline routes purely on+ // extension → parser family, and TCADP reconstructs by family.+ // The OLE fallback below therefore only applies to the local+ // office_oxide path.return parseWithTCADP(ctx,filename, data, strings.ToUpper(p.format),p.TCADPAPIServer, p.TCADPAPIKey,@@ -107,7 +118,27 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data// PDF-specific methods like "paddleocr" / "mineru" are// meaningless for PPTX; treat as default path.}- doc, err := officeOxide.OpenFromBytes(data, p.format)+ // office_oxide's OpenFromBytes takes the container format as given+ // and does no magic-byte detection, so a legacy .ppt uploaded with a+ // .pptx extension would be parsed as ZIP/OOXML and fail. Sniff the+ // real container and pass the matching format so mislabeled files+ // still parse (mirrors the magic-byte correction office_oxide::Open+ // performs on file paths).+ // File["format"] in the success case reflects effFormat, i.e. the+ // real container ("ppt" for OLE fallback, "pptx" for OOXML fallback),+ // not just the extension.+ effFormat := p.format+ switch officeContainer(data) {+ case "ole":+ if effFormat == "pptx" {+ effFormat = "ppt"+ }+ case "ooxml":+ if effFormat == "ppt" {+ effFormat = "pptx"+ }+ }+ doc, err := pptxOpenFromBytes(data, effFormat)if err != nil {return ParseResult{Err: fmt.Errorf("presentation open: %w", err)}}@@ -138,7 +169,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{OutputFormat: "json",- File: map[string]any{"name": filename, "format": p.format},+ File: map[string]any{"name": filename, "format": effFormat},JSON: items,}}