Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): recover invalid XLSX sheet names (#19264)

jay777211c3c6223291d
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index bcf881ad5e..3a6b430d99 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -480,7 +480,6 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
if !handledVision && !handledMedia && !handledImage && !handledAudio {
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)
- dispatched = hydrateEmptyDispatchPayload(dispatched, binary)
// Vision figure enhancement: on the JSON output path,
// append vision-model descriptions to embedded image and
@@ -508,7 +507,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
var pages [][]byte
var dispatchedPages []schema.Page
switch {
- case dispatched.Err == nil && dispatched.OutputFormat == "json" && len(dispatched.JSON) > 0:
+ case dispatched.Err == nil && dispatched.OutputFormat == "json":
dispatchedPages = jsonItemsToPages(dispatched.JSON)
pages = pagesFromDispatch(dispatchedPages)
case dispatched.Err == nil && dispatched.OutputFormat != "":
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 5dcbe30f90..69fb1c8510 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -469,41 +469,6 @@ func buildParserOutputs(parsed []schema.Page, dispatched parserDispatchResult, n
return out
}
-func hydrateEmptyDispatchPayload(dispatched parserDispatchResult, binary []byte) parserDispatchResult {
- if dispatched.Err != nil || len(binary) == 0 {
- return dispatched
- }
- switch dispatched.OutputFormat {
- case "json":
- if len(dispatched.JSON) == 0 {
- dispatched.JSON = pagesToJSONItems(splitIntoPages(binary))
- }
- case "text":
- if dispatched.Text == "" {
- dispatched.Text = string(binary)
- }
- }
- return dispatched
-}
-
-func pagesToJSONItems(pages [][]byte) []map[string]any {
- if len(pages) == 0 {
- pages = splitIntoPages(nil)
- }
- out := make([]map[string]any, 0, len(pages))
- for _, page := range pages {
- text := string(page)
- if text == "" {
- continue
- }
- out = append(out, map[string]any{
- "text": text,
- "doc_type_kwd": "text",
- })
- }
- return out
-}
-
func parserInputName(inputs map[string]any, docID string) string {
if inputs != nil {
if name, ok := inputs["name"].(string); ok && name != "" {
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index a24721bc5d..0d2900cc10 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -22,6 +22,7 @@ import (
"strings"
"testing"
+ "github.com/xuri/excelize/v2"
"ragflow/internal/agent/runtime"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
@@ -114,6 +115,41 @@ func TestParserComponent_Invoke_TextInput(t *testing.T) {
}
}
+func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {
+ f := excelize.NewFile()
+ data, err := f.WriteToBuffer()
+ if err != nil {
+ t.Fatalf("WriteToBuffer: %v", err)
+ }
+ if err := f.Close(); err != nil {
+ t.Fatalf("Close: %v", err)
+ }
+
+ c := &ParserComponent{Param: schema.ParserParam{}.Defaults(), Setups: defaultSetups()}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": data.Bytes(),
+ "file_type": "xlsx",
+ "name": "empty.xlsx",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ items, ok := out["json"].([]map[string]any)
+ if !ok {
+ t.Fatalf("json = %T, want []map[string]any", out["json"])
+ }
+ if len(items) != 0 {
+ t.Fatalf("empty XLSX JSON = %v, want no raw-binary items", items)
+ }
+ pages, ok := out["pages"].([]schema.Page)
+ if !ok {
+ t.Fatalf("pages = %T, want []schema.Page", out["pages"])
+ }
+ if len(pages) != 0 {
+ t.Fatalf("empty XLSX pages = %v, want none", pages)
+ }
+}
+
// TestParserComponent_Invoke_PageRangeFilter asserts that
// form-feed boundaries are honored: "A\fB\fC" yields three
// pages, in input order, with text intact.
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 3e63539362..dc50ba0c54 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -316,11 +316,11 @@ type mergeRange struct {
}
// mergeRanges returns the merged-cell rectangles of a sheet.
-func mergeRanges(f *excelize.File, sheet string) []mergeRange {
+func mergeRanges(f *excelize.File, sheet string) ([]mergeRange, error) {
var out []mergeRange
- cells, err := f.GetMergeCells(sheet)
+ cells, err := f.GetMergeCells(sheet, true)
if err != nil {
- return out
+ return nil, err
}
for _, mc := range cells {
sr, sc := axisToRC(mc.GetStartAxis())
@@ -330,7 +330,7 @@ func mergeRanges(f *excelize.File, sheet string) []mergeRange {
}
out = append(out, mergeRange{sr, sc, er, ec})
}
- return out
+ return out, nil
}
// mergeMaxCol returns the furthest merged column across all ranges, or 0 if
@@ -454,14 +454,14 @@ func cellIsStyled(f *excelize.File, sheet string, row, col int) bool {
// candidate that looks like a data row (majority numeric) is skipped. The
// override only applies when the anchor is not already row 1, so the common
// header-on-row-1 sheet is left unchanged.
-func detectHeaderRow(f *excelize.File, sheet string, records [][]string) int {
+func detectHeaderRow(f *excelize.File, sheet string, records [][]string, tables []excelize.Table) int {
n := len(records)
if n == 0 {
return 1
}
// 1) ListObject first.
- if tables, err := f.GetTables(sheet); err == nil && len(tables) > 0 {
+ if len(tables) > 0 {
minTop := 0
for _, t := range tables {
tr := rangeTopRow(t.Range)
@@ -637,26 +637,39 @@ func decodeChunkRows(setup map[string]any) int {
// renderSheetTables renders a single workbook sheet into one or more
// self-contained <table> chunks using the shared spreadsheet-HTML contract:
// detect the header row, inherit merged-master text into the header, and split
-// data into chunkRows-sized atomic tables each repeating the header. An empty
-// or unreadable sheet yields an empty string.
-func renderSheetTables(f *excelize.File, sheet string, chunkRows int) string {
- chunks := renderSheetTableChunks(f, sheet, chunkRows)
+// data into chunkRows-sized atomic tables each repeating the header.
+func renderSheetTables(f *excelize.File, sheet string, chunkRows int) (string, []string, error) {
+ chunks, warnings, err := renderSheetTableChunks(f, sheet, chunkRows)
+ if err != nil {
+ return "", warnings, err
+ }
parts := make([]string, len(chunks))
for i, ch := range chunks {
parts[i] = ch.HTML
}
- return strings.Join(parts, "")
+ return strings.Join(parts, ""), warnings, nil
}
-func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htmlTableChunk {
+func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {
rows, err := f.GetRows(sheet)
- if err != nil || len(rows) == 0 {
- return nil
+ if err != nil {
+ return nil, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)
+ }
+ if len(rows) == 0 {
+ return nil, nil, nil
}
rows = cleanIllegalControlChars(rows)
- ranges := mergeRanges(f, sheet)
- headerRow := detectHeaderRow(f, sheet, rows)
+ var warnings []string
+ ranges, err := mergeRanges(f, sheet)
+ if err != nil {
+ warnings = append(warnings, fmt.Sprintf("read XLSX sheet %q merged cells: %v", sheet, err))
+ }
+ tables, err := f.GetTables(sheet)
+ if err != nil {
+ warnings = append(warnings, fmt.Sprintf("read XLSX sheet %q table metadata: %v", sheet, err))
+ }
+ headerRow := detectHeaderRow(f, sheet, rows, tables)
// Inherit merged-master text into the header row. excelize's GetRows
// truncates each row at its last valued cell, so a merged slave beyond that
@@ -686,7 +699,7 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htm
chunks := recordsToHTMLTableChunkList(records, chunkRows, sheet, headerRow)
if len(chunks) == 0 || len(absDataRows) == 0 {
- return chunks
+ return chunks, warnings, nil
}
if chunkRows <= 0 {
chunkRows = defaultTableChunkRows
@@ -702,5 +715,5 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htm
chunks[ci].RowEnd = absDataRows[end-1]
}
}
- return chunks
+ return chunks, warnings, nil
}
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 84f9610291..e542d8c3ec 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -108,13 +108,20 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
var html strings.Builder
+ var warnings []string
for _, sheet := range sheets {
- html.WriteString(renderSheetTables(f, sheet, chunkRows))
+ rendered, sheetWarnings, err := renderSheetTables(f, sheet, chunkRows)
+ if err != nil {
+ return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
+ }
+ html.WriteString(rendered)
+ warnings = append(warnings, sheetWarnings...)
}
return ParseResult{
OutputFormat: "html",
File: map[string]any{"name": filename, "format": "xls"},
HTML: html.String(),
+ Warnings: warnings,
}
}
diff --git a/internal/parser/parser/xlsx_normalize.go b/internal/parser/parser/xlsx_normalize.go
new file mode 100644
index 0000000000..94f385cde8
--- /dev/null
+++ b/internal/parser/parser/xlsx_normalize.go
@@ -0,0 +1,381 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "strings"
+ "unicode/utf16"
+)
+
+const (
+ xlsxWorkbookXML = "xl/workbook.xml"
+ maxXLSXSheetNameUnits = 31
+)
+
+// normalizeXLSXForRead rewrites only invalid workbook sheet names so Excelize
+// can read otherwise intact sheet data. It leaves every non-workbook ZIP entry
+// untouched and returns changed=false when no name needs normalization.
+func normalizeXLSXForRead(data []byte) ([]byte, []string, bool, error) {
+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("open XLSX archive: %w", err)
+ }
+
+ var workbook []byte
+ for _, file := range src.File {
+ if file.Name != xlsxWorkbookXML {
+ continue
+ }
+ r, err := file.Open()
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("open %s: %w", xlsxWorkbookXML, err)
+ }
+ workbook, err = io.ReadAll(r)
+ closeErr := r.Close()
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("read %s: %w", xlsxWorkbookXML, err)
+ }
+ if closeErr != nil {
+ return nil, nil, false, fmt.Errorf("close %s: %w", xlsxWorkbookXML, closeErr)
+ }
+ break
+ }
+ if workbook == nil {
+ return nil, nil, false, fmt.Errorf("XLSX archive is missing %s", xlsxWorkbookXML)
+ }
+
+ normalizedWorkbook, warnings, changed, err := normalizeWorkbookSheetNames(workbook)
+ if err != nil || !changed {
+ return data, warnings, changed, err
+ }
+
+ var out bytes.Buffer
+ zw := zip.NewWriter(&out)
+ for _, file := range src.File {
+ w, err := zw.CreateHeader(&zip.FileHeader{Name: file.Name, Method: zip.Deflate})
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("write XLSX entry %q: %w", file.Name, err)
+ }
+ if file.Name == xlsxWorkbookXML {
+ if _, err := w.Write(normalizedWorkbook); err != nil {
+ return nil, nil, false, fmt.Errorf("write %s: %w", xlsxWorkbookXML, err)
+ }
+ continue
+ }
+ r, err := file.Open()
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("open XLSX entry %q: %w", file.Name, err)
+ }
+ _, copyErr := io.Copy(w, r)
+ closeErr := r.Close()
+ if copyErr != nil {
+ return nil, nil, false, fmt.Errorf("copy XLSX entry %q: %w", file.Name, copyErr)
+ }
+ if closeErr != nil {
+ return nil, nil, false, fmt.Errorf("close XLSX entry %q: %w", file.Name, closeErr)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ return nil, nil, false, fmt.Errorf("finish normalized XLSX archive: %w", err)
+ }
+ return out.Bytes(), warnings, true, nil
+}
+
+func normalizeWorkbookSheetNames(workbook []byte) ([]byte, []string, bool, error) {
+ var out bytes.Buffer
+ used := make([]string, 0)
+ changed := false
+ var warnings []string
+
+ lastWrite := 0
+ for offset := 0; offset < len(workbook); {
+ start := bytes.IndexByte(workbook[offset:], '<')
+ if start < 0 {
+ break
+ }
+ start += offset
+ if end, skipped, err := xmlNonElementEnd(workbook, start); err != nil {
+ return nil, nil, false, err
+ } else if skipped {
+ offset = end + 1
+ continue
+ }
+ end, err := xmlStartTagEnd(workbook, start)
+ if err != nil {
+ return nil, nil, false, err
+ }
+ tag := workbook[start : end+1]
+ if !isSheetStartTag(tag) {
+ offset = end + 1
+ continue
+ }
+
+ valueStart, valueEnd, quote, found, err := sheetNameAttributeBounds(tag)
+ if err != nil {
+ return nil, nil, false, err
+ }
+ if !found {
+ offset = end + 1
+ continue
+ }
+ original, err := decodeXMLAttributeValue(tag[valueStart:valueEnd], quote)
+ if err != nil {
+ return nil, nil, false, fmt.Errorf("decode XLSX sheet name: %w", err)
+ }
+ normalized := normalizedXLSXSheetName(original, used)
+ used = append(used, normalized)
+ if normalized != original {
+ out.Write(workbook[lastWrite:start])
+ out.Write(tag[:valueStart])
+ out.WriteString(encodeXMLAttributeValue(normalized, quote))
+ out.Write(tag[valueEnd:])
+ lastWrite = end + 1
+ offset = end + 1
+ changed = true
+ warnings = append(warnings, fmt.Sprintf("XLSX normalized sheet name %q to %q", original, normalized))
+ continue
+ }
+ offset = end + 1
+ }
+ if !changed {
+ return workbook, warnings, false, nil
+ }
+ out.Write(workbook[lastWrite:])
+ return out.Bytes(), warnings, true, nil
+}
+
+func xmlNonElementEnd(data []byte, start int) (int, bool, error) {
+ for _, marker := range []struct {
+ prefix []byte
+ suffix []byte
+ }{
+ {[]byte("<!--"), []byte("-->")},
+ {[]byte("<![CDATA["), []byte("]]>")},
+ {[]byte("<?"), []byte("?>")},
+ } {
+ if !bytes.HasPrefix(data[start:], marker.prefix) {
+ continue
+ }
+ end := bytes.Index(data[start+len(marker.prefix):], marker.suffix)
+ if end < 0 {
+ return 0, false, fmt.Errorf("scan workbook XML: unterminated markup at byte %d", start)
+ }
+ return start + len(marker.prefix) + end + len(marker.suffix) - 1, true, nil
+ }
+ return 0, false, nil
+}
+
+func xmlStartTagEnd(data []byte, start int) (int, error) {
+ var quote byte
+ for i := start + 1; i < len(data); i++ {
+ switch data[i] {
+ case '\'', '"':
+ if quote == 0 {
+ quote = data[i]
+ } else if quote == data[i] {
+ quote = 0
+ }
+ case '>':
+ if quote == 0 {
+ return i, nil
+ }
+ }
+ }
+ return 0, fmt.Errorf("scan workbook XML: unterminated tag at byte %d", start)
+}
+
+func isSheetStartTag(tag []byte) bool {
+ if len(tag) < len("<sheet") || tag[0] != '<' {
+ return false
+ }
+ i := 1
+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '/' && tag[i] != '>' {
+ i++
+ }
+ name := tag[1:i]
+ if colon := bytes.LastIndexByte(name, ':'); colon >= 0 {
+ name = name[colon+1:]
+ }
+ return bytes.Equal(name, []byte("sheet"))
+}
+
+func sheetNameAttributeBounds(tag []byte) (int, int, byte, bool, error) {
+ i := 1
+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '/' && tag[i] != '>' {
+ i++
+ }
+ for i < len(tag) {
+ for i < len(tag) && isXMLSpace(tag[i]) {
+ i++
+ }
+ if i >= len(tag) || tag[i] == '/' || tag[i] == '>' {
+ return 0, 0, 0, false, nil
+ }
+ nameStart := i
+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '=' && tag[i] != '/' && tag[i] != '>' {
+ i++
+ }
+ name := tag[nameStart:i]
+ for i < len(tag) && isXMLSpace(tag[i]) {
+ i++
+ }
+ if i >= len(tag) || tag[i] != '=' {
+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: invalid sheet attribute %q", name)
+ }
+ i++
+ for i < len(tag) && isXMLSpace(tag[i]) {
+ i++
+ }
+ if i >= len(tag) || (tag[i] != '\'' && tag[i] != '"') {
+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: unquoted sheet attribute %q", name)
+ }
+ quote := tag[i]
+ i++
+ valueStart := i
+ for i < len(tag) && tag[i] != quote {
+ i++
+ }
+ if i >= len(tag) {
+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: unterminated sheet attribute %q", name)
+ }
+ valueEnd := i
+ i++
+ if xmlLocalName(name) == "name" {
+ return valueStart, valueEnd, quote, true, nil
+ }
+ }
+ return 0, 0, 0, false, nil
+}
+
+func decodeXMLAttributeValue(value []byte, quote byte) (string, error) {
+ var attr struct {
+ Value string `xml:"value,attr"`
+ }
+ encoded := make([]byte, 0, len(value)+20)
+ encoded = append(encoded, "<sheet value="...)
+ encoded = append(encoded, quote)
+ encoded = append(encoded, value...)
+ encoded = append(encoded, quote, '/', '>')
+ if err := xml.Unmarshal(encoded, &attr); err != nil {
+ return "", err
+ }
+ return attr.Value, nil
+}
+
+func encodeXMLAttributeValue(value string, quote byte) string {
+ var out strings.Builder
+ for _, r := range value {
+ switch r {
+ case '&':
+ out.WriteString("&amp;")
+ case '<':
+ out.WriteString("&lt;")
+ case '>':
+ out.WriteString("&gt;")
+ case '\'':
+ if quote == '\'' {
+ out.WriteString("&apos;")
+ } else {
+ out.WriteRune(r)
+ }
+ case '"':
+ if quote == '"' {
+ out.WriteString("&quot;")
+ } else {
+ out.WriteRune(r)
+ }
+ case '\t':
+ out.WriteString("&#x9;")
+ case '\n':
+ out.WriteString("&#xA;")
+ case '\r':
+ out.WriteString("&#xD;")
+ default:
+ out.WriteRune(r)
+ }
+ }
+ return out.String()
+}
+
+func isXMLSpace(b byte) bool {
+ return b == ' ' || b == '\t' || b == '\n' || b == '\r'
+}
+
+func xmlLocalName(name []byte) string {
+ if colon := bytes.LastIndexByte(name, ':'); colon >= 0 {
+ name = name[colon+1:]
+ }
+ return string(name)
+}
+
+func normalizedXLSXSheetName(name string, used []string) string {
+ var b strings.Builder
+ for _, r := range strings.Trim(name, "'") {
+ switch r {
+ case ':', '\\', '/', '?', '*', '[', ']':
+ b.WriteByte('_')
+ default:
+ b.WriteRune(r)
+ }
+ }
+ base := truncateSheetName(b.String(), maxXLSXSheetNameUnits)
+ if base == "" {
+ base = "Sheet"
+ }
+ name = base
+ for suffix := 2; ; suffix++ {
+ if !containsEqualFold(used, name) {
+ return name
+ }
+ tail := fmt.Sprintf("_%d", suffix)
+ name = truncateSheetName(base, maxXLSXSheetNameUnits-utf16UnitCount(tail)) + tail
+ }
+}
+
+func containsEqualFold(names []string, name string) bool {
+ for _, candidate := range names {
+ if strings.EqualFold(candidate, name) {
+ return true
+ }
+ }
+ return false
+}
+
+func truncateSheetName(name string, maxUnits int) string {
+ units := 0
+ for i, r := range name {
+ width := utf16.RuneLen(r)
+ if units+width > maxUnits {
+ return name[:i]
+ }
+ units += width
+ }
+ return name
+}
+
+func utf16UnitCount(name string) int {
+ units := 0
+ for _, r := range name {
+ units += utf16.RuneLen(r)
+ }
+ return units
+}
diff --git a/internal/parser/parser/xlsx_normalize_test.go b/internal/parser/parser/xlsx_normalize_test.go
new file mode 100644
index 0000000000..381909d42b
--- /dev/null
+++ b/internal/parser/parser/xlsx_normalize_test.go
@@ -0,0 +1,265 @@
+package parser
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/xml"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/xuri/excelize/v2"
+)
+
+func TestXLSXParser_NormalizesInvalidSheetName(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "B1", "Amount")
+ mustSetCell(t, f, "Sheet1", "A2", "Alice")
+ mustSetCell(t, f, "Sheet1", "B2", "100")
+ })
+ data = xlsxWithWorkbookSheetName(t, data, "Visible:Data")
+
+ p, err := NewXLSXParser("")
+ if err != nil {
+ t.Fatalf("NewXLSXParser: %v", err)
+ }
+ res := p.ParseWithResult(t.Context(), "invalid-sheet-name.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+
+ html := xlsxTableHTML(res)
+ for _, want := range []string{"Name", "Amount", "Alice", "100"} {
+ if !strings.Contains(html, want) {
+ t.Fatalf("parsed table = %q, want %q", html, want)
+ }
+ }
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "Visible:Data") {
+ t.Fatalf("warnings = %v, want sheet-name normalization warning", res.Warnings)
+ }
+}
+
+func TestNormalizeXLSXForReadPreservesWorkbookXMLOutsideSheetName(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ })
+ data = xlsxWithWorkbookSheetName(t, data, "Visible:Data")
+ original := readXLSXEntry(t, data, xlsxWorkbookXML)
+
+ normalized, _, changed, err := normalizeXLSXForRead(data)
+ if err != nil {
+ t.Fatalf("normalizeXLSXForRead: %v", err)
+ }
+ if !changed {
+ t.Fatal("normalizeXLSXForRead: want changed workbook")
+ }
+
+ want := strings.Replace(string(original), `name="Visible:Data"`, `name="Visible_Data"`, 1)
+ if got := string(readXLSXEntry(t, normalized, xlsxWorkbookXML)); got != want {
+ t.Fatalf("normalized workbook.xml changed bytes outside sheet name:\n got: %s\nwant: %s", got, want)
+ }
+}
+
+func TestXLSXParser_NormalizesEmojiSheetNameWithinUTF16Limit(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "A2", "Alice")
+ })
+ data = xlsxWithWorkbookSheetName(t, data, strings.Repeat("😀", 31)+":")
+
+ p, err := NewXLSXParser("")
+ if err != nil {
+ t.Fatalf("NewXLSXParser: %v", err)
+ }
+ res := p.ParseWithResult(t.Context(), "emoji-sheet-name.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if !strings.Contains(xlsxTableHTML(res), "Alice") {
+ t.Fatalf("parsed table = %q, want Alice", xlsxTableHTML(res))
+ }
+}
+
+func TestNormalizeWorkbookSheetNamesAvoidsEqualFoldCollisions(t *testing.T) {
+ workbook := []byte(`<workbook><sheets><sheet name="'Σ'"/><sheet name="ς"/></sheets></workbook>`)
+ normalized, _, changed, err := normalizeWorkbookSheetNames(workbook)
+ if err != nil {
+ t.Fatalf("normalizeWorkbookSheetNames: %v", err)
+ }
+ if !changed {
+ t.Fatal("normalizeWorkbookSheetNames: want changed workbook")
+ }
+
+ var result struct {
+ Sheets []struct {
+ Name string `xml:"name,attr"`
+ } `xml:"sheets>sheet"`
+ }
+ if err := xml.Unmarshal(normalized, &result); err != nil {
+ t.Fatalf("Unmarshal normalized workbook: %v", err)
+ }
+ if len(result.Sheets) != 2 {
+ t.Fatalf("normalized sheets = %d, want 2", len(result.Sheets))
+ }
+ if strings.EqualFold(result.Sheets[0].Name, result.Sheets[1].Name) {
+ t.Fatalf("normalized sheet names %q and %q collide under EqualFold", result.Sheets[0].Name, result.Sheets[1].Name)
+ }
+}
+
+func TestXLSXParser_ReturnsUnrecoverableReadError(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ })
+ data = xlsxWithWorksheetXML(t, data, `<?xml version="1.0" encoding="UTF-8"?><worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1048577"><c r="A1048577" t="inlineStr"><is><t>invalid row</t></is></c></row></sheetData></worksheet>`)
+
+ p, err := NewXLSXParser("")
+ if err != nil {
+ t.Fatalf("NewXLSXParser: %v", err)
+ }
+ res := p.ParseWithResult(t.Context(), "invalid-row.xlsx", data)
+ if res.Err == nil {
+ t.Fatal("ParseWithResult: want read error, got nil")
+ }
+ if !strings.Contains(res.Err.Error(), "row number exceeds") {
+ t.Fatalf("error = %q, want row-number context", res.Err)
+ }
+}
+
+func TestXLSXParser_WarnsWhenMergedCellsCannotBeRead(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "A2", "Alice")
+ })
+ data = xlsxWithWorksheetXML(t, data, `<?xml version="1.0" encoding="UTF-8"?><worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Name</t></is></c></row><row r="2"><c r="A2" t="inlineStr"><is><t>Alice</t></is></c></row></sheetData><mergeCells count="1"><mergeCell ref="not-a-range"/></mergeCells></worksheet>`)
+
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "bad-merge.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if !strings.Contains(xlsxTableHTML(res), "Alice") {
+ t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))
+ }
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "merged cells") {
+ t.Fatalf("warnings = %v, want merged-cells warning", res.Warnings)
+ }
+}
+
+func TestXLSXParser_WarnsWhenTableMetadataCannotBeRead(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Name")
+ mustSetCell(t, f, "Sheet1", "A2", "Alice")
+ mustAddTable(t, f, "Sheet1", &excelize.Table{Range: "A1:A2", Name: "People"})
+ })
+ data = rewriteXLSXEntry(t, data, "xl/tables/table1.xml", func([]byte) []byte {
+ return []byte(`<table`)
+ })
+
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "bad-table.xml", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if !strings.Contains(xlsxTableHTML(res), "Alice") {
+ t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))
+ }
+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "table metadata") {
+ t.Fatalf("warnings = %v, want table-metadata warning", res.Warnings)
+ }
+}
+
+func xlsxWithWorkbookSheetName(t *testing.T, data []byte, name string) []byte {
+ return rewriteXLSXEntry(t, data, "xl/workbook.xml", func(content []byte) []byte {
+ updated := strings.Replace(string(content), `name="Sheet1"`, `name="`+name+`"`, 1)
+ if updated == string(content) {
+ t.Fatal("workbook.xml did not contain Sheet1")
+ }
+ return []byte(updated)
+ })
+}
+
+func xlsxWithWorksheetXML(t *testing.T, data []byte, worksheet string) []byte {
+ return rewriteXLSXEntry(t, data, "xl/worksheets/sheet1.xml", func([]byte) []byte {
+ return []byte(worksheet)
+ })
+}
+
+func readXLSXEntry(t *testing.T, data []byte, entry string) []byte {
+ t.Helper()
+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatalf("open source xlsx: %v", err)
+ }
+ for _, file := range src.File {
+ if file.Name != entry {
+ continue
+ }
+ r, err := file.Open()
+ if err != nil {
+ t.Fatalf("open %s: %v", entry, err)
+ }
+ content, err := io.ReadAll(r)
+ closeErr := r.Close()
+ if err != nil {
+ t.Fatalf("read %s: %v", entry, err)
+ }
+ if closeErr != nil {
+ t.Fatalf("close %s: %v", entry, closeErr)
+ }
+ return content
+ }
+ t.Fatalf("XLSX entry %q not found", entry)
+ return nil
+}
+
+func rewriteXLSXEntry(t *testing.T, data []byte, entry string, rewrite func([]byte) []byte) []byte {
+ t.Helper()
+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatalf("open source xlsx: %v", err)
+ }
+
+ var out bytes.Buffer
+ zw := zip.NewWriter(&out)
+ for _, file := range src.File {
+ w, err := zw.CreateHeader(&zip.FileHeader{Name: file.Name, Method: zip.Deflate})
+ if err != nil {
+ t.Fatalf("create %s: %v", file.Name, err)
+ }
+ if file.Name == entry {
+ r, err := file.Open()
+ if err != nil {
+ t.Fatalf("open %s: %v", entry, err)
+ }
+ content, err := io.ReadAll(r)
+ closeErr := r.Close()
+ if err != nil {
+ t.Fatalf("read %s: %v", entry, err)
+ }
+ if closeErr != nil {
+ t.Fatalf("close %s: %v", entry, closeErr)
+ }
+ if _, err := w.Write(rewrite(content)); err != nil {
+ t.Fatalf("write %s: %v", entry, err)
+ }
+ continue
+ }
+ r, err := file.Open()
+ if err != nil {
+ t.Fatalf("open %s: %v", file.Name, err)
+ }
+ _, err = io.Copy(w, r)
+ closeErr := r.Close()
+ if err != nil {
+ t.Fatalf("copy %s: %v", file.Name, err)
+ }
+ if closeErr != nil {
+ t.Fatalf("close %s: %v", file.Name, closeErr)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatalf("close xlsx: %v", err)
+ }
+ return out.Bytes()
+}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index fa16b6c398..e370623a91 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -113,22 +113,48 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
// for spreadsheet processing.
}
+ chunkRows := p.ChunkRows
+ if chunkRows <= 0 {
+ chunkRows = defaultTableChunkRows
+ }
+
+ items, warnings, sheets, err := parseXLSXBytes(data, chunkRows)
+ if err == nil {
+ return xlsxParseResult(filename, items, warnings, sheets)
+ }
+
+ normalized, normalizeWarnings, changed, normalizeErr := normalizeXLSXForRead(data)
+ if normalizeErr != nil {
+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w; normalize: %v", err, normalizeErr)}
+ }
+ if !changed {
+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}
+ }
+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, chunkRows)
+ if retryErr != nil {
+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
+ }
+ warnings = append(normalizeWarnings, warnings...)
+ return xlsxParseResult(filename, items, warnings, sheets)
+}
+
+func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
- return ParseResult{Err: fmt.Errorf("xlsx open: %w", err)}
+ return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
}
defer f.Close()
sheets := f.GetSheetList()
- chunkRows := p.ChunkRows
- if chunkRows <= 0 {
- chunkRows = defaultTableChunkRows
- }
-
items := make([]map[string]any, 0)
warnings := make([]string, 0)
for sheetIdx, sheet := range sheets {
- for _, table := range renderSheetTableChunks(f, sheet, chunkRows) {
+ tables, sheetWarnings, err := renderSheetTableChunks(f, sheet, chunkRows)
+ if err != nil {
+ return nil, warnings, len(sheets), err
+ }
+ warnings = append(warnings, sheetWarnings...)
+ for _, table := range tables {
items = append(items, map[string]any{
"text": table.HTML,
"doc_type_kwd": "table",
@@ -147,10 +173,13 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
items = append(items, images...)
warnings = append(warnings, imageWarnings...)
}
+ return items, warnings, len(sheets), nil
+}
+func xlsxParseResult(filename string, items []map[string]any, warnings []string, sheets int) ParseResult {
return ParseResult{
OutputFormat: "json",
- File: map[string]any{"name": filename, "format": "xlsx", "sheets": len(sheets)},
+ File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},
JSON: items,
Warnings: warnings,
}