Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(data-analysis): read legacy xls workbooks (#4218)

AsanoReo5ca9cabb65e2
diff --git a/go.mod b/go.mod
index ca3282c17..9be637150 100644
--- a/go.mod
+++ b/go.mod
@@ -51,6 +51,7 @@ require (
github.com/redis/go-redis/v9 v9.14.1
github.com/robfig/cron/v3 v3.0.1
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
+ github.com/shakinm/xlsReader v0.9.12
github.com/sirupsen/logrus v1.9.4
github.com/slack-go/slack v0.23.1
github.com/spf13/viper v1.21.0
@@ -92,6 +93,8 @@ require (
gorm.io/gorm v1.31.1
)
+require github.com/metakeule/fmtdate v1.1.2 // indirect
+
require (
cloud.google.com/go/auth v0.20.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
diff --git a/go.sum b/go.sum
index ec0253068..72cede3bc 100644
--- a/go.sum
+++ b/go.sum
@@ -3135,6 +3135,8 @@ github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7c
github.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs=
github.com/mediocregopher/radix/v3 v3.4.2 h1:galbPBjIwmyREgwGCfQEN4X8lxbJnKBYurgz+VfcStA=
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
+github.com/metakeule/fmtdate v1.1.2 h1:n9M7H9HfAqp+6OA98wXGMdcAr6omshSNVct65Bks1lQ=
+github.com/metakeule/fmtdate v1.1.2/go.mod h1:2JyMFlKxeoGy1qS6obQukT0AL0Y4iNANQL8scbSdT4E=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
@@ -3530,6 +3532,8 @@ github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvK
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
+github.com/shakinm/xlsReader v0.9.12 h1:F6GWYtCzfzQqdIuqZJ0MU3YJ7uwH1ofJtmTKyWmANQk=
+github.com/shakinm/xlsReader v0.9.12/go.mod h1:ME9pqIGf+547L4aE4YTZzwmhsij+5K9dR+k84OO6WSs=
github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY=
diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go
index 3b41a7aff..852df8657 100644
--- a/internal/agent/tools/data_analysis.go
+++ b/internal/agent/tools/data_analysis.go
@@ -37,9 +37,9 @@ var dataAnalysisTool = BaseTool{
"the single table \"" + DataAnalysisTableName + "\"; write SQL against that table name and never put " +
"the document ID inside the SQL. For Excel files with multiple sheets, non-empty sheets are loaded into " +
"the same table and the source sheet name is exposed as a '__sheet_name' column so you can " +
- "filter/aggregate per sheet. Only the .xlsx container is readable: a legacy binary .xls must be " +
- "re-saved as .xlsx (or exported as .csv) before it can be analysed. If the user's question requires " +
- "data statistics, convert the question into SQL and execute it.",
+ "filter/aggregate per sheet. A legacy binary .xls is converted to .xlsx before it is read; " +
+ "if that conversion fails, re-save the workbook as .xlsx or export the sheet as .csv. " +
+ "If the user's question requires data statistics, convert the question into SQL and execute it.",
schema: utils.GenerateSchema[DataAnalysisInput](),
}
@@ -51,9 +51,9 @@ const excelSheetNameColumn = "__sheet_name"
// .xls workbooks start with ("D0 CF 11 E0 A1 B1 1A E1").
//
// DuckDB's excel extension only reads the ZIP-based OOXML container
-// (.xlsx/.xlsm), so a genuine .xls has to be rejected up front: handed to
-// read_xlsx it fails with the opaque "No xl/workbook.xml found in xlsx file",
-// which says nothing about the real cause or what to do about it.
+// (.xlsx/.xlsm). A genuine .xls is converted before read_xlsx; if that
+// conversion fails, the caller gets an actionable error instead of
+// "No xl/workbook.xml found in xlsx file".
var legacyXLSMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
// hasLegacyXLSContainer reports whether path holds an OLE2 compound document,
@@ -77,10 +77,9 @@ func hasLegacyXLSContainer(path string) bool {
return bytes.Equal(head, legacyXLSMagic)
}
-// errLegacyBinaryXLS is the actionable failure returned for a binary .xls
-// workbook. The data analysis tool cannot read it, and silently reporting an
-// empty table (or leaking DuckDB's xlsx-internals error) leaves the model
-// guessing; naming the conversion to perform is what the caller can act on.
+// errLegacyBinaryXLS is returned when a binary .xls workbook cannot be
+// converted to .xlsx. Naming the conversion the caller can still do beats
+// DuckDB's "No xl/workbook.xml found in xlsx file".
func errLegacyBinaryXLS() error {
return fmt.Errorf(
"legacy binary .xls workbooks are not supported by data analysis: " +
@@ -592,23 +591,27 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab
// Note: requires the DuckDB 'excel' extension (for read_xlsx) and the
// 'spatial' extension (for st_read_meta used to enumerate sheets).
//
-// Binary BIFF .xls workbooks are rejected with an actionable error before
-// DuckDB is touched: the excel extension only reads the ZIP-based OOXML
-// container, so a real .xls would otherwise surface as "No xl/workbook.xml
-// found in xlsx file".
+// Binary BIFF .xls workbooks are converted to a temporary .xlsx before DuckDB
+// is touched. The excel extension only reads the ZIP-based OOXML container, so
+// a real .xls would otherwise surface as "No xl/workbook.xml found in xlsx
+// file". A workbook that cannot be converted returns an actionable error.
func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) {
logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)
- if hasLegacyXLSContainer(filename) {
- logger.Warnf(ctx,
- "[Tool][DataAnalysis] Rejecting legacy binary .xls workbook '%s' (session=%s): "+
- "DuckDB can only read the ZIP-based .xlsx container",
- filename, t.sessionID,
- )
- return nil, errLegacyBinaryXLS()
- }
-
if !t.loadedTables[tableName] {
+ if hasLegacyXLSContainer(filename) {
+ prepared, cleanup, err := prepareExcelForDuckDB(filename)
+ if err != nil {
+ logger.Warnf(ctx,
+ "[Tool][DataAnalysis] Could not convert legacy binary .xls workbook '%s' (session=%s): %v",
+ filename, t.sessionID, err,
+ )
+ return nil, errLegacyBinaryXLS()
+ }
+ defer cleanup()
+ filename = prepared
+ }
+
t.recordCreatedTable(tableName)
sheetNames, enumErr := t.listExcelSheets(ctx, filename)
if enumErr != nil {
diff --git a/internal/agent/tools/data_analysis_xls.go b/internal/agent/tools/data_analysis_xls.go
new file mode 100644
index 000000000..64f4f3e6b
--- /dev/null
+++ b/internal/agent/tools/data_analysis_xls.go
@@ -0,0 +1,156 @@
+package tools
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/shakinm/xlsReader/xls"
+ "github.com/xuri/excelize/v2"
+)
+
+var oleCompoundHeader = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}
+
+// prepareExcelForDuckDB returns a path DuckDB's read_xlsx can open.
+// A legacy OLE workbook is converted to a temporary .xlsx; the cleanup
+// function removes that file. Anything else, including a .xls that is
+// really a renamed .xlsx, is returned unchanged.
+func prepareExcelForDuckDB(filename string) (string, func(), error) {
+ noop := func() {}
+ ole, err := isOLECompoundFile(filename)
+ if err != nil {
+ return "", noop, err
+ }
+ if !ole {
+ return filename, noop, nil
+ }
+ converted, err := convertLegacyXLSToXLSX(filename)
+ if err != nil {
+ return "", noop, err
+ }
+ cleanup := func() {
+ _ = os.Remove(converted)
+ }
+ return converted, cleanup, nil
+}
+
+func isOLECompoundFile(filename string) (bool, error) {
+ f, err := os.Open(filename)
+ if err != nil {
+ return false, err
+ }
+ defer func() { _ = f.Close() }()
+ header := make([]byte, len(oleCompoundHeader))
+ if _, err := io.ReadFull(f, header); err != nil {
+ if err == io.EOF || err == io.ErrUnexpectedEOF {
+ return false, nil
+ }
+ return false, err
+ }
+ return bytes.Equal(header, oleCompoundHeader), nil
+}
+
+func convertLegacyXLSToXLSX(filename string) (string, error) {
+ workbook, err := xls.OpenFile(filename)
+ if err != nil {
+ return "", fmt.Errorf("read xls workbook: %w", err)
+ }
+ out := excelize.NewFile()
+ defer func() { _ = out.Close() }()
+
+ used := map[string]struct{}{}
+ wrote := false
+ for i := 0; i < workbook.GetNumberSheets(); i++ {
+ sheet, err := workbook.GetSheet(i)
+ if err != nil || sheet == nil {
+ continue
+ }
+ name := uniqueExcelSheetName(sheet.GetName(), i, used)
+ if !wrote {
+ if err := out.SetSheetName("Sheet1", name); err != nil {
+ return "", err
+ }
+ wrote = true
+ } else if _, err := out.NewSheet(name); err != nil {
+ return "", err
+ }
+ for r := 0; r < sheet.GetNumberRows(); r++ {
+ row, err := sheet.GetRow(r)
+ if err != nil || row == nil {
+ continue
+ }
+ for c, col := range row.GetCols() {
+ if col == nil {
+ continue
+ }
+ value := col.GetString()
+ if value == "" {
+ continue
+ }
+ cell, err := excelize.CoordinatesToCellName(c+1, r+1)
+ if err != nil {
+ return "", err
+ }
+ if err := out.SetCellValue(name, cell, value); err != nil {
+ return "", err
+ }
+ }
+ }
+ }
+ if !wrote {
+ return "", fmt.Errorf("xls workbook %s has no sheets", filename)
+ }
+
+ tmp, err := os.CreateTemp("", "weknora-xls-*.xlsx")
+ if err != nil {
+ return "", err
+ }
+ tmpPath := tmp.Name()
+ if err := tmp.Close(); err != nil {
+ _ = os.Remove(tmpPath)
+ return "", err
+ }
+ if err := out.SaveAs(tmpPath); err != nil {
+ _ = os.Remove(tmpPath)
+ return "", err
+ }
+ return tmpPath, nil
+}
+
+func uniqueExcelSheetName(name string, index int, used map[string]struct{}) string {
+ name = strings.Map(func(r rune) rune {
+ switch r {
+ case ':', '\\', '/', '?', '*', '[', ']':
+ return '_'
+ default:
+ return r
+ }
+ }, strings.TrimSpace(name))
+ if name == "" {
+ name = fmt.Sprintf("Sheet%d", index+1)
+ }
+ name = trimRunes(name, 31)
+ base := name
+ for n := 2; ; n++ {
+ if _, ok := used[name]; !ok {
+ used[name] = struct{}{}
+ return name
+ }
+ suffix := fmt.Sprintf("_%d", n)
+ name = trimRunes(base, 31-utf8.RuneCountInString(suffix)) + suffix
+ }
+}
+
+func trimRunes(s string, n int) string {
+ if n <= 0 {
+ return ""
+ }
+ runes := []rune(s)
+ if len(runes) <= n {
+ return s
+ }
+ return string(runes[:n])
+}
diff --git a/internal/agent/tools/data_analysis_xls_test.go b/internal/agent/tools/data_analysis_xls_test.go
new file mode 100644
index 000000000..65cd31f4c
--- /dev/null
+++ b/internal/agent/tools/data_analysis_xls_test.go
@@ -0,0 +1,97 @@
+package tools
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "github.com/xuri/excelize/v2"
+)
+
+func TestPrepareExcelForDuckDBConvertsLegacyXLS(t *testing.T) {
+ src := filepath.Join("testdata", "legacy.xls")
+ converted, cleanup, err := prepareExcelForDuckDB(src)
+ require.NoError(t, err)
+ t.Cleanup(cleanup)
+ require.NotEqual(t, src, converted)
+
+ book, err := excelize.OpenFile(converted)
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = book.Close() })
+
+ sales, err := book.GetRows("销售")
+ require.NoError(t, err)
+ require.Equal(t, []string{"名称", "数量"}, sales[0])
+ require.Equal(t, []string{"苹果", "3"}, sales[1])
+ require.Equal(t, []string{"梨", "12.5"}, sales[2])
+
+ stock, err := book.GetRows("库存")
+ require.NoError(t, err)
+ require.Equal(t, [][]string{{"sku"}, {"A-1"}}, stock)
+}
+
+func TestPrepareExcelForDuckDBLeavesNonOLEXlsAlone(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "not.xls")
+ require.NoError(t, os.WriteFile(path, []byte("PK\x03\x04not-an-xls"), 0o644))
+ got, cleanup, err := prepareExcelForDuckDB(path)
+ require.NoError(t, err)
+ cleanup()
+ require.Equal(t, path, got)
+}
+
+func TestPrepareExcelForDuckDBAcceptsUppercaseExtension(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("testdata", "legacy.xls"))
+ require.NoError(t, err)
+ path := filepath.Join(t.TempDir(), "LEGACY.XLS")
+ require.NoError(t, os.WriteFile(path, data, 0o644))
+
+ converted, cleanup, err := prepareExcelForDuckDB(path)
+ require.NoError(t, err)
+ t.Cleanup(cleanup)
+
+ book, err := excelize.OpenFile(converted)
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = book.Close() })
+ rows, err := book.GetRows("销售")
+ require.NoError(t, err)
+ require.Equal(t, []string{"苹果", "3"}, rows[1])
+}
+
+func TestUniqueExcelSheetNameStripsReservedCharacters(t *testing.T) {
+ used := map[string]struct{}{}
+ first := uniqueExcelSheetName("a/b:c", 0, used)
+ second := uniqueExcelSheetName("a/b:c", 1, used)
+ require.Equal(t, "a_b_c", first)
+ require.NotEqual(t, first, second)
+ require.LessOrEqual(t, len([]rune(second)), 31)
+}
+
+func TestPrepareExcelForDuckDBLeavesXLSXAlone(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "book.xlsx")
+ writeWorkbook(t, path, map[string][][]any{
+ "Sheet1": {{"a"}, {"b"}},
+ }, []string{"Sheet1"})
+
+ got, cleanup, err := prepareExcelForDuckDB(path)
+ require.NoError(t, err)
+ cleanup()
+ require.Equal(t, path, got)
+}
+
+func TestLoadFromExcelReadsLegacyXLS(t *testing.T) {
+ db := newTestDuckDB(t)
+ tool := &DataAnalysisTool{
+ BaseTool: dataAnalysisTool,
+ db: db,
+ sessionID: "test-legacy-xls",
+ }
+ ctx := context.Background()
+ schema, err := tool.LoadFromExcel(ctx, filepath.Join("testdata", "legacy.xls"), "t_legacy")
+ require.NoError(t, err)
+ t.Cleanup(func() { tool.Cleanup(ctx) })
+
+ // Two data rows on 销售 and one on 库存.
+ require.Equal(t, int64(3), schema.RowCount)
+}
diff --git a/internal/agent/tools/testdata/legacy.xls b/internal/agent/tools/testdata/legacy.xls
new file mode 100644
index 000000000..0d638c57e
Binary files /dev/null and b/internal/agent/tools/testdata/legacy.xls differ