RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): recover invalid XLSX sheet names (#19264)
jay777211c3c6223291d
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex bcf881ad5e..3a6b430d99 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -480,7 +480,6 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}if !handledVision && !handledMedia && !handledImage && !handledAudio {dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, c.Setups)- dispatched = hydrateEmptyDispatchPayload(dispatched, binary)// Vision figure enhancement: on the JSON output path,// append vision-model descriptions to embedded image and@@ -508,7 +507,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stvar pages [][]bytevar dispatchedPages []schema.Pageswitch {- case dispatched.Err == nil && dispatched.OutputFormat == "json" && len(dispatched.JSON) > 0:+ case dispatched.Err == nil && dispatched.OutputFormat == "json":dispatchedPages = jsonItemsToPages(dispatched.JSON)pages = pagesFromDispatch(dispatchedPages)case dispatched.Err == nil && dispatched.OutputFormat != "":diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 5dcbe30f90..69fb1c8510 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -469,41 +469,6 @@ func buildParserOutputs(parsed []schema.Page, dispatched parserDispatchResult, nreturn out}-func hydrateEmptyDispatchPayload(dispatched parserDispatchResult, binary []byte) parserDispatchResult {- if dispatched.Err != nil || len(binary) == 0 {- return dispatched- }- switch dispatched.OutputFormat {- case "json":- if len(dispatched.JSON) == 0 {- dispatched.JSON = pagesToJSONItems(splitIntoPages(binary))- }- case "text":- if dispatched.Text == "" {- dispatched.Text = string(binary)- }- }- return dispatched-}--func pagesToJSONItems(pages [][]byte) []map[string]any {- if len(pages) == 0 {- pages = splitIntoPages(nil)- }- out := make([]map[string]any, 0, len(pages))- for _, page := range pages {- text := string(page)- if text == "" {- continue- }- out = append(out, map[string]any{- "text": text,- "doc_type_kwd": "text",- })- }- return out-}-func parserInputName(inputs map[string]any, docID string) string {if inputs != nil {if name, ok := inputs["name"].(string); ok && name != "" {diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex a24721bc5d..0d2900cc10 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -22,6 +22,7 @@ import ("strings""testing"+ "github.com/xuri/excelize/v2""ragflow/internal/agent/runtime""ragflow/internal/entity""ragflow/internal/ingestion/component/schema"@@ -114,6 +115,41 @@ func TestParserComponent_Invoke_TextInput(t *testing.T) {}}+func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {+ f := excelize.NewFile()+ data, err := f.WriteToBuffer()+ if err != nil {+ t.Fatalf("WriteToBuffer: %v", err)+ }+ if err := f.Close(); err != nil {+ t.Fatalf("Close: %v", err)+ }++ c := &ParserComponent{Param: schema.ParserParam{}.Defaults(), Setups: defaultSetups()}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "binary": data.Bytes(),+ "file_type": "xlsx",+ "name": "empty.xlsx",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ items, ok := out["json"].([]map[string]any)+ if !ok {+ t.Fatalf("json = %T, want []map[string]any", out["json"])+ }+ if len(items) != 0 {+ t.Fatalf("empty XLSX JSON = %v, want no raw-binary items", items)+ }+ pages, ok := out["pages"].([]schema.Page)+ if !ok {+ t.Fatalf("pages = %T, want []schema.Page", out["pages"])+ }+ if len(pages) != 0 {+ t.Fatalf("empty XLSX pages = %v, want none", pages)+ }+}+// TestParserComponent_Invoke_PageRangeFilter asserts that// form-feed boundaries are honored: "A\fB\fC" yields three// pages, in input order, with text intact.diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 3e63539362..dc50ba0c54 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -316,11 +316,11 @@ type mergeRange struct {}// mergeRanges returns the merged-cell rectangles of a sheet.-func mergeRanges(f *excelize.File, sheet string) []mergeRange {+func mergeRanges(f *excelize.File, sheet string) ([]mergeRange, error) {var out []mergeRange- cells, err := f.GetMergeCells(sheet)+ cells, err := f.GetMergeCells(sheet, true)if err != nil {- return out+ return nil, err}for _, mc := range cells {sr, sc := axisToRC(mc.GetStartAxis())@@ -330,7 +330,7 @@ func mergeRanges(f *excelize.File, sheet string) []mergeRange {}out = append(out, mergeRange{sr, sc, er, ec})}- return out+ return out, nil}// mergeMaxCol returns the furthest merged column across all ranges, or 0 if@@ -454,14 +454,14 @@ func cellIsStyled(f *excelize.File, sheet string, row, col int) bool {// candidate that looks like a data row (majority numeric) is skipped. The// override only applies when the anchor is not already row 1, so the common// header-on-row-1 sheet is left unchanged.-func detectHeaderRow(f *excelize.File, sheet string, records [][]string) int {+func detectHeaderRow(f *excelize.File, sheet string, records [][]string, tables []excelize.Table) int {n := len(records)if n == 0 {return 1}// 1) ListObject first.- if tables, err := f.GetTables(sheet); err == nil && len(tables) > 0 {+ if len(tables) > 0 {minTop := 0for _, t := range tables {tr := rangeTopRow(t.Range)@@ -637,26 +637,39 @@ func decodeChunkRows(setup map[string]any) int {// renderSheetTables renders a single workbook sheet into one or more// self-contained <table> chunks using the shared spreadsheet-HTML contract:// detect the header row, inherit merged-master text into the header, and split-// data into chunkRows-sized atomic tables each repeating the header. An empty-// or unreadable sheet yields an empty string.-func renderSheetTables(f *excelize.File, sheet string, chunkRows int) string {- chunks := renderSheetTableChunks(f, sheet, chunkRows)+// data into chunkRows-sized atomic tables each repeating the header.+func renderSheetTables(f *excelize.File, sheet string, chunkRows int) (string, []string, error) {+ chunks, warnings, err := renderSheetTableChunks(f, sheet, chunkRows)+ if err != nil {+ return "", warnings, err+ }parts := make([]string, len(chunks))for i, ch := range chunks {parts[i] = ch.HTML}- return strings.Join(parts, "")+ return strings.Join(parts, ""), warnings, nil}-func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htmlTableChunk {+func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {rows, err := f.GetRows(sheet)- if err != nil || len(rows) == 0 {- return nil+ if err != nil {+ return nil, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)+ }+ if len(rows) == 0 {+ return nil, nil, nil}rows = cleanIllegalControlChars(rows)- ranges := mergeRanges(f, sheet)- headerRow := detectHeaderRow(f, sheet, rows)+ var warnings []string+ ranges, err := mergeRanges(f, sheet)+ if err != nil {+ warnings = append(warnings, fmt.Sprintf("read XLSX sheet %q merged cells: %v", sheet, err))+ }+ tables, err := f.GetTables(sheet)+ if err != nil {+ warnings = append(warnings, fmt.Sprintf("read XLSX sheet %q table metadata: %v", sheet, err))+ }+ headerRow := detectHeaderRow(f, sheet, rows, tables)// Inherit merged-master text into the header row. excelize's GetRows// truncates each row at its last valued cell, so a merged slave beyond that@@ -686,7 +699,7 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htmchunks := recordsToHTMLTableChunkList(records, chunkRows, sheet, headerRow)if len(chunks) == 0 || len(absDataRows) == 0 {- return chunks+ return chunks, warnings, nil}if chunkRows <= 0 {chunkRows = defaultTableChunkRows@@ -702,5 +715,5 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) []htmchunks[ci].RowEnd = absDataRows[end-1]}}- return chunks+ return chunks, warnings, nil}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 84f9610291..e542d8c3ec 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -108,13 +108,20 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [}var html strings.Builder+ var warnings []stringfor _, sheet := range sheets {- html.WriteString(renderSheetTables(f, sheet, chunkRows))+ rendered, sheetWarnings, err := renderSheetTables(f, sheet, chunkRows)+ if err != nil {+ return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}+ }+ html.WriteString(rendered)+ warnings = append(warnings, sheetWarnings...)}return ParseResult{OutputFormat: "html",File: map[string]any{"name": filename, "format": "xls"},HTML: html.String(),+ Warnings: warnings,}}diff --git a/internal/parser/parser/xlsx_normalize.go b/internal/parser/parser/xlsx_normalize.gonew file mode 100644index 0000000000..94f385cde8--- /dev/null+++ b/internal/parser/parser/xlsx_normalize.go@@ -0,0 +1,381 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++import (+ "archive/zip"+ "bytes"+ "encoding/xml"+ "fmt"+ "io"+ "strings"+ "unicode/utf16"+)++const (+ xlsxWorkbookXML = "xl/workbook.xml"+ maxXLSXSheetNameUnits = 31+)++// normalizeXLSXForRead rewrites only invalid workbook sheet names so Excelize+// can read otherwise intact sheet data. It leaves every non-workbook ZIP entry+// untouched and returns changed=false when no name needs normalization.+func normalizeXLSXForRead(data []byte) ([]byte, []string, bool, error) {+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ return nil, nil, false, fmt.Errorf("open XLSX archive: %w", err)+ }++ var workbook []byte+ for _, file := range src.File {+ if file.Name != xlsxWorkbookXML {+ continue+ }+ r, err := file.Open()+ if err != nil {+ return nil, nil, false, fmt.Errorf("open %s: %w", xlsxWorkbookXML, err)+ }+ workbook, err = io.ReadAll(r)+ closeErr := r.Close()+ if err != nil {+ return nil, nil, false, fmt.Errorf("read %s: %w", xlsxWorkbookXML, err)+ }+ if closeErr != nil {+ return nil, nil, false, fmt.Errorf("close %s: %w", xlsxWorkbookXML, closeErr)+ }+ break+ }+ if workbook == nil {+ return nil, nil, false, fmt.Errorf("XLSX archive is missing %s", xlsxWorkbookXML)+ }++ normalizedWorkbook, warnings, changed, err := normalizeWorkbookSheetNames(workbook)+ if err != nil || !changed {+ return data, warnings, changed, err+ }++ var out bytes.Buffer+ zw := zip.NewWriter(&out)+ for _, file := range src.File {+ w, err := zw.CreateHeader(&zip.FileHeader{Name: file.Name, Method: zip.Deflate})+ if err != nil {+ return nil, nil, false, fmt.Errorf("write XLSX entry %q: %w", file.Name, err)+ }+ if file.Name == xlsxWorkbookXML {+ if _, err := w.Write(normalizedWorkbook); err != nil {+ return nil, nil, false, fmt.Errorf("write %s: %w", xlsxWorkbookXML, err)+ }+ continue+ }+ r, err := file.Open()+ if err != nil {+ return nil, nil, false, fmt.Errorf("open XLSX entry %q: %w", file.Name, err)+ }+ _, copyErr := io.Copy(w, r)+ closeErr := r.Close()+ if copyErr != nil {+ return nil, nil, false, fmt.Errorf("copy XLSX entry %q: %w", file.Name, copyErr)+ }+ if closeErr != nil {+ return nil, nil, false, fmt.Errorf("close XLSX entry %q: %w", file.Name, closeErr)+ }+ }+ if err := zw.Close(); err != nil {+ return nil, nil, false, fmt.Errorf("finish normalized XLSX archive: %w", err)+ }+ return out.Bytes(), warnings, true, nil+}++func normalizeWorkbookSheetNames(workbook []byte) ([]byte, []string, bool, error) {+ var out bytes.Buffer+ used := make([]string, 0)+ changed := false+ var warnings []string++ lastWrite := 0+ for offset := 0; offset < len(workbook); {+ start := bytes.IndexByte(workbook[offset:], '<')+ if start < 0 {+ break+ }+ start += offset+ if end, skipped, err := xmlNonElementEnd(workbook, start); err != nil {+ return nil, nil, false, err+ } else if skipped {+ offset = end + 1+ continue+ }+ end, err := xmlStartTagEnd(workbook, start)+ if err != nil {+ return nil, nil, false, err+ }+ tag := workbook[start : end+1]+ if !isSheetStartTag(tag) {+ offset = end + 1+ continue+ }++ valueStart, valueEnd, quote, found, err := sheetNameAttributeBounds(tag)+ if err != nil {+ return nil, nil, false, err+ }+ if !found {+ offset = end + 1+ continue+ }+ original, err := decodeXMLAttributeValue(tag[valueStart:valueEnd], quote)+ if err != nil {+ return nil, nil, false, fmt.Errorf("decode XLSX sheet name: %w", err)+ }+ normalized := normalizedXLSXSheetName(original, used)+ used = append(used, normalized)+ if normalized != original {+ out.Write(workbook[lastWrite:start])+ out.Write(tag[:valueStart])+ out.WriteString(encodeXMLAttributeValue(normalized, quote))+ out.Write(tag[valueEnd:])+ lastWrite = end + 1+ offset = end + 1+ changed = true+ warnings = append(warnings, fmt.Sprintf("XLSX normalized sheet name %q to %q", original, normalized))+ continue+ }+ offset = end + 1+ }+ if !changed {+ return workbook, warnings, false, nil+ }+ out.Write(workbook[lastWrite:])+ return out.Bytes(), warnings, true, nil+}++func xmlNonElementEnd(data []byte, start int) (int, bool, error) {+ for _, marker := range []struct {+ prefix []byte+ suffix []byte+ }{+ {[]byte("<!--"), []byte("-->")},+ {[]byte("<![CDATA["), []byte("]]>")},+ {[]byte("<?"), []byte("?>")},+ } {+ if !bytes.HasPrefix(data[start:], marker.prefix) {+ continue+ }+ end := bytes.Index(data[start+len(marker.prefix):], marker.suffix)+ if end < 0 {+ return 0, false, fmt.Errorf("scan workbook XML: unterminated markup at byte %d", start)+ }+ return start + len(marker.prefix) + end + len(marker.suffix) - 1, true, nil+ }+ return 0, false, nil+}++func xmlStartTagEnd(data []byte, start int) (int, error) {+ var quote byte+ for i := start + 1; i < len(data); i++ {+ switch data[i] {+ case '\'', '"':+ if quote == 0 {+ quote = data[i]+ } else if quote == data[i] {+ quote = 0+ }+ case '>':+ if quote == 0 {+ return i, nil+ }+ }+ }+ return 0, fmt.Errorf("scan workbook XML: unterminated tag at byte %d", start)+}++func isSheetStartTag(tag []byte) bool {+ if len(tag) < len("<sheet") || tag[0] != '<' {+ return false+ }+ i := 1+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '/' && tag[i] != '>' {+ i+++ }+ name := tag[1:i]+ if colon := bytes.LastIndexByte(name, ':'); colon >= 0 {+ name = name[colon+1:]+ }+ return bytes.Equal(name, []byte("sheet"))+}++func sheetNameAttributeBounds(tag []byte) (int, int, byte, bool, error) {+ i := 1+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '/' && tag[i] != '>' {+ i+++ }+ for i < len(tag) {+ for i < len(tag) && isXMLSpace(tag[i]) {+ i+++ }+ if i >= len(tag) || tag[i] == '/' || tag[i] == '>' {+ return 0, 0, 0, false, nil+ }+ nameStart := i+ for i < len(tag) && !isXMLSpace(tag[i]) && tag[i] != '=' && tag[i] != '/' && tag[i] != '>' {+ i+++ }+ name := tag[nameStart:i]+ for i < len(tag) && isXMLSpace(tag[i]) {+ i+++ }+ if i >= len(tag) || tag[i] != '=' {+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: invalid sheet attribute %q", name)+ }+ i+++ for i < len(tag) && isXMLSpace(tag[i]) {+ i+++ }+ if i >= len(tag) || (tag[i] != '\'' && tag[i] != '"') {+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: unquoted sheet attribute %q", name)+ }+ quote := tag[i]+ i+++ valueStart := i+ for i < len(tag) && tag[i] != quote {+ i+++ }+ if i >= len(tag) {+ return 0, 0, 0, false, fmt.Errorf("scan workbook XML: unterminated sheet attribute %q", name)+ }+ valueEnd := i+ i+++ if xmlLocalName(name) == "name" {+ return valueStart, valueEnd, quote, true, nil+ }+ }+ return 0, 0, 0, false, nil+}++func decodeXMLAttributeValue(value []byte, quote byte) (string, error) {+ var attr struct {+ Value string `xml:"value,attr"`+ }+ encoded := make([]byte, 0, len(value)+20)+ encoded = append(encoded, "<sheet value="...)+ encoded = append(encoded, quote)+ encoded = append(encoded, value...)+ encoded = append(encoded, quote, '/', '>')+ if err := xml.Unmarshal(encoded, &attr); err != nil {+ return "", err+ }+ return attr.Value, nil+}++func encodeXMLAttributeValue(value string, quote byte) string {+ var out strings.Builder+ for _, r := range value {+ switch r {+ case '&':+ out.WriteString("&")+ case '<':+ out.WriteString("<")+ case '>':+ out.WriteString(">")+ case '\'':+ if quote == '\'' {+ out.WriteString("'")+ } else {+ out.WriteRune(r)+ }+ case '"':+ if quote == '"' {+ out.WriteString(""")+ } else {+ out.WriteRune(r)+ }+ case '\t':+ out.WriteString("	")+ case '\n':+ out.WriteString("
")+ case '\r':+ out.WriteString("
")+ default:+ out.WriteRune(r)+ }+ }+ return out.String()+}++func isXMLSpace(b byte) bool {+ return b == ' ' || b == '\t' || b == '\n' || b == '\r'+}++func xmlLocalName(name []byte) string {+ if colon := bytes.LastIndexByte(name, ':'); colon >= 0 {+ name = name[colon+1:]+ }+ return string(name)+}++func normalizedXLSXSheetName(name string, used []string) string {+ var b strings.Builder+ for _, r := range strings.Trim(name, "'") {+ switch r {+ case ':', '\\', '/', '?', '*', '[', ']':+ b.WriteByte('_')+ default:+ b.WriteRune(r)+ }+ }+ base := truncateSheetName(b.String(), maxXLSXSheetNameUnits)+ if base == "" {+ base = "Sheet"+ }+ name = base+ for suffix := 2; ; suffix++ {+ if !containsEqualFold(used, name) {+ return name+ }+ tail := fmt.Sprintf("_%d", suffix)+ name = truncateSheetName(base, maxXLSXSheetNameUnits-utf16UnitCount(tail)) + tail+ }+}++func containsEqualFold(names []string, name string) bool {+ for _, candidate := range names {+ if strings.EqualFold(candidate, name) {+ return true+ }+ }+ return false+}++func truncateSheetName(name string, maxUnits int) string {+ units := 0+ for i, r := range name {+ width := utf16.RuneLen(r)+ if units+width > maxUnits {+ return name[:i]+ }+ units += width+ }+ return name+}++func utf16UnitCount(name string) int {+ units := 0+ for _, r := range name {+ units += utf16.RuneLen(r)+ }+ return units+}diff --git a/internal/parser/parser/xlsx_normalize_test.go b/internal/parser/parser/xlsx_normalize_test.gonew file mode 100644index 0000000000..381909d42b--- /dev/null+++ b/internal/parser/parser/xlsx_normalize_test.go@@ -0,0 +1,265 @@+package parser++import (+ "archive/zip"+ "bytes"+ "encoding/xml"+ "io"+ "strings"+ "testing"++ "github.com/xuri/excelize/v2"+)++func TestXLSXParser_NormalizesInvalidSheetName(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "B1", "Amount")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ mustSetCell(t, f, "Sheet1", "B2", "100")+ })+ data = xlsxWithWorkbookSheetName(t, data, "Visible:Data")++ p, err := NewXLSXParser("")+ if err != nil {+ t.Fatalf("NewXLSXParser: %v", err)+ }+ res := p.ParseWithResult(t.Context(), "invalid-sheet-name.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }++ html := xlsxTableHTML(res)+ for _, want := range []string{"Name", "Amount", "Alice", "100"} {+ if !strings.Contains(html, want) {+ t.Fatalf("parsed table = %q, want %q", html, want)+ }+ }+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "Visible:Data") {+ t.Fatalf("warnings = %v, want sheet-name normalization warning", res.Warnings)+ }+}++func TestNormalizeXLSXForReadPreservesWorkbookXMLOutsideSheetName(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ })+ data = xlsxWithWorkbookSheetName(t, data, "Visible:Data")+ original := readXLSXEntry(t, data, xlsxWorkbookXML)++ normalized, _, changed, err := normalizeXLSXForRead(data)+ if err != nil {+ t.Fatalf("normalizeXLSXForRead: %v", err)+ }+ if !changed {+ t.Fatal("normalizeXLSXForRead: want changed workbook")+ }++ want := strings.Replace(string(original), `name="Visible:Data"`, `name="Visible_Data"`, 1)+ if got := string(readXLSXEntry(t, normalized, xlsxWorkbookXML)); got != want {+ t.Fatalf("normalized workbook.xml changed bytes outside sheet name:\n got: %s\nwant: %s", got, want)+ }+}++func TestXLSXParser_NormalizesEmojiSheetNameWithinUTF16Limit(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ })+ data = xlsxWithWorkbookSheetName(t, data, strings.Repeat("😀", 31)+":")++ p, err := NewXLSXParser("")+ if err != nil {+ t.Fatalf("NewXLSXParser: %v", err)+ }+ res := p.ParseWithResult(t.Context(), "emoji-sheet-name.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if !strings.Contains(xlsxTableHTML(res), "Alice") {+ t.Fatalf("parsed table = %q, want Alice", xlsxTableHTML(res))+ }+}++func TestNormalizeWorkbookSheetNamesAvoidsEqualFoldCollisions(t *testing.T) {+ workbook := []byte(`<workbook><sheets><sheet name="'Σ'"/><sheet name="ς"/></sheets></workbook>`)+ normalized, _, changed, err := normalizeWorkbookSheetNames(workbook)+ if err != nil {+ t.Fatalf("normalizeWorkbookSheetNames: %v", err)+ }+ if !changed {+ t.Fatal("normalizeWorkbookSheetNames: want changed workbook")+ }++ var result struct {+ Sheets []struct {+ Name string `xml:"name,attr"`+ } `xml:"sheets>sheet"`+ }+ if err := xml.Unmarshal(normalized, &result); err != nil {+ t.Fatalf("Unmarshal normalized workbook: %v", err)+ }+ if len(result.Sheets) != 2 {+ t.Fatalf("normalized sheets = %d, want 2", len(result.Sheets))+ }+ if strings.EqualFold(result.Sheets[0].Name, result.Sheets[1].Name) {+ t.Fatalf("normalized sheet names %q and %q collide under EqualFold", result.Sheets[0].Name, result.Sheets[1].Name)+ }+}++func TestXLSXParser_ReturnsUnrecoverableReadError(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ })+ data = xlsxWithWorksheetXML(t, data, `<?xml version="1.0" encoding="UTF-8"?><worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1048577"><c r="A1048577" t="inlineStr"><is><t>invalid row</t></is></c></row></sheetData></worksheet>`)++ p, err := NewXLSXParser("")+ if err != nil {+ t.Fatalf("NewXLSXParser: %v", err)+ }+ res := p.ParseWithResult(t.Context(), "invalid-row.xlsx", data)+ if res.Err == nil {+ t.Fatal("ParseWithResult: want read error, got nil")+ }+ if !strings.Contains(res.Err.Error(), "row number exceeds") {+ t.Fatalf("error = %q, want row-number context", res.Err)+ }+}++func TestXLSXParser_WarnsWhenMergedCellsCannotBeRead(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ })+ data = xlsxWithWorksheetXML(t, data, `<?xml version="1.0" encoding="UTF-8"?><worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Name</t></is></c></row><row r="2"><c r="A2" t="inlineStr"><is><t>Alice</t></is></c></row></sheetData><mergeCells count="1"><mergeCell ref="not-a-range"/></mergeCells></worksheet>`)++ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "bad-merge.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if !strings.Contains(xlsxTableHTML(res), "Alice") {+ t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))+ }+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "merged cells") {+ t.Fatalf("warnings = %v, want merged-cells warning", res.Warnings)+ }+}++func TestXLSXParser_WarnsWhenTableMetadataCannotBeRead(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Name")+ mustSetCell(t, f, "Sheet1", "A2", "Alice")+ mustAddTable(t, f, "Sheet1", &excelize.Table{Range: "A1:A2", Name: "People"})+ })+ data = rewriteXLSXEntry(t, data, "xl/tables/table1.xml", func([]byte) []byte {+ return []byte(`<table`)+ })++ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "bad-table.xml", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if !strings.Contains(xlsxTableHTML(res), "Alice") {+ t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))+ }+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "table metadata") {+ t.Fatalf("warnings = %v, want table-metadata warning", res.Warnings)+ }+}++func xlsxWithWorkbookSheetName(t *testing.T, data []byte, name string) []byte {+ return rewriteXLSXEntry(t, data, "xl/workbook.xml", func(content []byte) []byte {+ updated := strings.Replace(string(content), `name="Sheet1"`, `name="`+name+`"`, 1)+ if updated == string(content) {+ t.Fatal("workbook.xml did not contain Sheet1")+ }+ return []byte(updated)+ })+}++func xlsxWithWorksheetXML(t *testing.T, data []byte, worksheet string) []byte {+ return rewriteXLSXEntry(t, data, "xl/worksheets/sheet1.xml", func([]byte) []byte {+ return []byte(worksheet)+ })+}++func readXLSXEntry(t *testing.T, data []byte, entry string) []byte {+ t.Helper()+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ t.Fatalf("open source xlsx: %v", err)+ }+ for _, file := range src.File {+ if file.Name != entry {+ continue+ }+ r, err := file.Open()+ if err != nil {+ t.Fatalf("open %s: %v", entry, err)+ }+ content, err := io.ReadAll(r)+ closeErr := r.Close()+ if err != nil {+ t.Fatalf("read %s: %v", entry, err)+ }+ if closeErr != nil {+ t.Fatalf("close %s: %v", entry, closeErr)+ }+ return content+ }+ t.Fatalf("XLSX entry %q not found", entry)+ return nil+}++func rewriteXLSXEntry(t *testing.T, data []byte, entry string, rewrite func([]byte) []byte) []byte {+ t.Helper()+ src, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ t.Fatalf("open source xlsx: %v", err)+ }++ var out bytes.Buffer+ zw := zip.NewWriter(&out)+ for _, file := range src.File {+ w, err := zw.CreateHeader(&zip.FileHeader{Name: file.Name, Method: zip.Deflate})+ if err != nil {+ t.Fatalf("create %s: %v", file.Name, err)+ }+ if file.Name == entry {+ r, err := file.Open()+ if err != nil {+ t.Fatalf("open %s: %v", entry, err)+ }+ content, err := io.ReadAll(r)+ closeErr := r.Close()+ if err != nil {+ t.Fatalf("read %s: %v", entry, err)+ }+ if closeErr != nil {+ t.Fatalf("close %s: %v", entry, closeErr)+ }+ if _, err := w.Write(rewrite(content)); err != nil {+ t.Fatalf("write %s: %v", entry, err)+ }+ continue+ }+ r, err := file.Open()+ if err != nil {+ t.Fatalf("open %s: %v", file.Name, err)+ }+ _, err = io.Copy(w, r)+ closeErr := r.Close()+ if err != nil {+ t.Fatalf("copy %s: %v", file.Name, err)+ }+ if closeErr != nil {+ t.Fatalf("close %s: %v", file.Name, closeErr)+ }+ }+ if err := zw.Close(); err != nil {+ t.Fatalf("close xlsx: %v", err)+ }+ return out.Bytes()+}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex fa16b6c398..e370623a91 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -113,22 +113,48 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}+ chunkRows := p.ChunkRows+ if chunkRows <= 0 {+ chunkRows = defaultTableChunkRows+ }++ items, warnings, sheets, err := parseXLSXBytes(data, chunkRows)+ if err == nil {+ return xlsxParseResult(filename, items, warnings, sheets)+ }++ normalized, normalizeWarnings, changed, normalizeErr := normalizeXLSXForRead(data)+ if normalizeErr != nil {+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w; normalize: %v", err, normalizeErr)}+ }+ if !changed {+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}+ }+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, chunkRows)+ if retryErr != nil {+ return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}+ }+ warnings = append(normalizeWarnings, warnings...)+ return xlsxParseResult(filename, items, warnings, sheets)+}++func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {- return ParseResult{Err: fmt.Errorf("xlsx open: %w", err)}+ return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)}defer f.Close()sheets := f.GetSheetList()- chunkRows := p.ChunkRows- if chunkRows <= 0 {- chunkRows = defaultTableChunkRows- }-items := make([]map[string]any, 0)warnings := make([]string, 0)for sheetIdx, sheet := range sheets {- for _, table := range renderSheetTableChunks(f, sheet, chunkRows) {+ tables, sheetWarnings, err := renderSheetTableChunks(f, sheet, chunkRows)+ if err != nil {+ return nil, warnings, len(sheets), err+ }+ warnings = append(warnings, sheetWarnings...)+ for _, table := range tables {items = append(items, map[string]any{"text": table.HTML,"doc_type_kwd": "table",@@ -147,10 +173,13 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, dataitems = append(items, images...)warnings = append(warnings, imageWarnings...)}+ return items, warnings, len(sheets), nil+}+func xlsxParseResult(filename string, items []map[string]any, warnings []string, sheets int) ParseResult {return ParseResult{OutputFormat: "json",- File: map[string]any{"name": filename, "format": "xlsx", "sheets": len(sheets)},+ File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},JSON: items,Warnings: warnings,}