Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(data-analysis): read legacy xls workbooks (#4218)
AsanoReo5ca9cabb65e2
diff --git a/go.mod b/go.modindex ca3282c17..9be637150 100644--- a/go.mod+++ b/go.mod@@ -51,6 +51,7 @@ require (github.com/redis/go-redis/v9 v9.14.1github.com/robfig/cron/v3 v3.0.1github.com/santhosh-tekuri/jsonschema/v6 v6.0.2+ github.com/shakinm/xlsReader v0.9.12github.com/sirupsen/logrus v1.9.4github.com/slack-go/slack v0.23.1github.com/spf13/viper v1.21.0@@ -92,6 +93,8 @@ require (gorm.io/gorm v1.31.1)+require github.com/metakeule/fmtdate v1.1.2 // indirect+require (cloud.google.com/go/auth v0.20.0 // indirectcloud.google.com/go/auth/oauth2adapt v0.2.8 // indirectdiff --git a/go.sum b/go.sumindex ec0253068..72cede3bc 100644--- a/go.sum+++ b/go.sum@@ -3135,6 +3135,8 @@ github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7cgithub.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs=github.com/mediocregopher/radix/v3 v3.4.2 h1:galbPBjIwmyREgwGCfQEN4X8lxbJnKBYurgz+VfcStA=github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=+github.com/metakeule/fmtdate v1.1.2 h1:n9M7H9HfAqp+6OA98wXGMdcAr6omshSNVct65Bks1lQ=+github.com/metakeule/fmtdate v1.1.2/go.mod h1:2JyMFlKxeoGy1qS6obQukT0AL0Y4iNANQL8scbSdT4E=github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=@@ -3530,6 +3532,8 @@ github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKgithub.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=+github.com/shakinm/xlsReader v0.9.12 h1:F6GWYtCzfzQqdIuqZJ0MU3YJ7uwH1ofJtmTKyWmANQk=+github.com/shakinm/xlsReader v0.9.12/go.mod h1:ME9pqIGf+547L4aE4YTZzwmhsij+5K9dR+k84OO6WSs=github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY=diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.goindex 3b41a7aff..852df8657 100644--- a/internal/agent/tools/data_analysis.go+++ b/internal/agent/tools/data_analysis.go@@ -37,9 +37,9 @@ var dataAnalysisTool = BaseTool{"the single table \"" + DataAnalysisTableName + "\"; write SQL against that table name and never put " +"the document ID inside the SQL. For Excel files with multiple sheets, non-empty sheets are loaded into " +"the same table and the source sheet name is exposed as a '__sheet_name' column so you can " +- "filter/aggregate per sheet. Only the .xlsx container is readable: a legacy binary .xls must be " +- "re-saved as .xlsx (or exported as .csv) before it can be analysed. If the user's question requires " +- "data statistics, convert the question into SQL and execute it.",+ "filter/aggregate per sheet. A legacy binary .xls is converted to .xlsx before it is read; " ++ "if that conversion fails, re-save the workbook as .xlsx or export the sheet as .csv. " ++ "If the user's question requires data statistics, convert the question into SQL and execute it.",schema: utils.GenerateSchema[DataAnalysisInput](),}@@ -51,9 +51,9 @@ const excelSheetNameColumn = "__sheet_name"// .xls workbooks start with ("D0 CF 11 E0 A1 B1 1A E1").//// DuckDB's excel extension only reads the ZIP-based OOXML container-// (.xlsx/.xlsm), so a genuine .xls has to be rejected up front: handed to-// read_xlsx it fails with the opaque "No xl/workbook.xml found in xlsx file",-// which says nothing about the real cause or what to do about it.+// (.xlsx/.xlsm). A genuine .xls is converted before read_xlsx; if that+// conversion fails, the caller gets an actionable error instead of+// "No xl/workbook.xml found in xlsx file".var legacyXLSMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}// hasLegacyXLSContainer reports whether path holds an OLE2 compound document,@@ -77,10 +77,9 @@ func hasLegacyXLSContainer(path string) bool {return bytes.Equal(head, legacyXLSMagic)}-// errLegacyBinaryXLS is the actionable failure returned for a binary .xls-// workbook. The data analysis tool cannot read it, and silently reporting an-// empty table (or leaking DuckDB's xlsx-internals error) leaves the model-// guessing; naming the conversion to perform is what the caller can act on.+// errLegacyBinaryXLS is returned when a binary .xls workbook cannot be+// converted to .xlsx. Naming the conversion the caller can still do beats+// DuckDB's "No xl/workbook.xml found in xlsx file".func errLegacyBinaryXLS() error {return fmt.Errorf("legacy binary .xls workbooks are not supported by data analysis: " +@@ -592,23 +591,27 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab// Note: requires the DuckDB 'excel' extension (for read_xlsx) and the// 'spatial' extension (for st_read_meta used to enumerate sheets).//-// Binary BIFF .xls workbooks are rejected with an actionable error before-// DuckDB is touched: the excel extension only reads the ZIP-based OOXML-// container, so a real .xls would otherwise surface as "No xl/workbook.xml-// found in xlsx file".+// Binary BIFF .xls workbooks are converted to a temporary .xlsx before DuckDB+// is touched. The excel extension only reads the ZIP-based OOXML container, so+// a real .xls would otherwise surface as "No xl/workbook.xml found in xlsx+// file". A workbook that cannot be converted returns an actionable error.func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) {logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)- if hasLegacyXLSContainer(filename) {- logger.Warnf(ctx,- "[Tool][DataAnalysis] Rejecting legacy binary .xls workbook '%s' (session=%s): "+- "DuckDB can only read the ZIP-based .xlsx container",- filename, t.sessionID,- )- return nil, errLegacyBinaryXLS()- }-if !t.loadedTables[tableName] {+ if hasLegacyXLSContainer(filename) {+ prepared, cleanup, err := prepareExcelForDuckDB(filename)+ if err != nil {+ logger.Warnf(ctx,+ "[Tool][DataAnalysis] Could not convert legacy binary .xls workbook '%s' (session=%s): %v",+ filename, t.sessionID, err,+ )+ return nil, errLegacyBinaryXLS()+ }+ defer cleanup()+ filename = prepared+ }+t.recordCreatedTable(tableName)sheetNames, enumErr := t.listExcelSheets(ctx, filename)if enumErr != nil {diff --git a/internal/agent/tools/data_analysis_xls.go b/internal/agent/tools/data_analysis_xls.gonew file mode 100644index 000000000..64f4f3e6b--- /dev/null+++ b/internal/agent/tools/data_analysis_xls.go@@ -0,0 +1,156 @@+package tools++import (+ "bytes"+ "fmt"+ "io"+ "os"+ "strings"+ "unicode/utf8"++ "github.com/shakinm/xlsReader/xls"+ "github.com/xuri/excelize/v2"+)++var oleCompoundHeader = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}++// prepareExcelForDuckDB returns a path DuckDB's read_xlsx can open.+// A legacy OLE workbook is converted to a temporary .xlsx; the cleanup+// function removes that file. Anything else, including a .xls that is+// really a renamed .xlsx, is returned unchanged.+func prepareExcelForDuckDB(filename string) (string, func(), error) {+ noop := func() {}+ ole, err := isOLECompoundFile(filename)+ if err != nil {+ return "", noop, err+ }+ if !ole {+ return filename, noop, nil+ }+ converted, err := convertLegacyXLSToXLSX(filename)+ if err != nil {+ return "", noop, err+ }+ cleanup := func() {+ _ = os.Remove(converted)+ }+ return converted, cleanup, nil+}++func isOLECompoundFile(filename string) (bool, error) {+ f, err := os.Open(filename)+ if err != nil {+ return false, err+ }+ defer func() { _ = f.Close() }()+ header := make([]byte, len(oleCompoundHeader))+ if _, err := io.ReadFull(f, header); err != nil {+ if err == io.EOF || err == io.ErrUnexpectedEOF {+ return false, nil+ }+ return false, err+ }+ return bytes.Equal(header, oleCompoundHeader), nil+}++func convertLegacyXLSToXLSX(filename string) (string, error) {+ workbook, err := xls.OpenFile(filename)+ if err != nil {+ return "", fmt.Errorf("read xls workbook: %w", err)+ }+ out := excelize.NewFile()+ defer func() { _ = out.Close() }()++ used := map[string]struct{}{}+ wrote := false+ for i := 0; i < workbook.GetNumberSheets(); i++ {+ sheet, err := workbook.GetSheet(i)+ if err != nil || sheet == nil {+ continue+ }+ name := uniqueExcelSheetName(sheet.GetName(), i, used)+ if !wrote {+ if err := out.SetSheetName("Sheet1", name); err != nil {+ return "", err+ }+ wrote = true+ } else if _, err := out.NewSheet(name); err != nil {+ return "", err+ }+ for r := 0; r < sheet.GetNumberRows(); r++ {+ row, err := sheet.GetRow(r)+ if err != nil || row == nil {+ continue+ }+ for c, col := range row.GetCols() {+ if col == nil {+ continue+ }+ value := col.GetString()+ if value == "" {+ continue+ }+ cell, err := excelize.CoordinatesToCellName(c+1, r+1)+ if err != nil {+ return "", err+ }+ if err := out.SetCellValue(name, cell, value); err != nil {+ return "", err+ }+ }+ }+ }+ if !wrote {+ return "", fmt.Errorf("xls workbook %s has no sheets", filename)+ }++ tmp, err := os.CreateTemp("", "weknora-xls-*.xlsx")+ if err != nil {+ return "", err+ }+ tmpPath := tmp.Name()+ if err := tmp.Close(); err != nil {+ _ = os.Remove(tmpPath)+ return "", err+ }+ if err := out.SaveAs(tmpPath); err != nil {+ _ = os.Remove(tmpPath)+ return "", err+ }+ return tmpPath, nil+}++func uniqueExcelSheetName(name string, index int, used map[string]struct{}) string {+ name = strings.Map(func(r rune) rune {+ switch r {+ case ':', '\\', '/', '?', '*', '[', ']':+ return '_'+ default:+ return r+ }+ }, strings.TrimSpace(name))+ if name == "" {+ name = fmt.Sprintf("Sheet%d", index+1)+ }+ name = trimRunes(name, 31)+ base := name+ for n := 2; ; n++ {+ if _, ok := used[name]; !ok {+ used[name] = struct{}{}+ return name+ }+ suffix := fmt.Sprintf("_%d", n)+ name = trimRunes(base, 31-utf8.RuneCountInString(suffix)) + suffix+ }+}++func trimRunes(s string, n int) string {+ if n <= 0 {+ return ""+ }+ runes := []rune(s)+ if len(runes) <= n {+ return s+ }+ return string(runes[:n])+}diff --git a/internal/agent/tools/data_analysis_xls_test.go b/internal/agent/tools/data_analysis_xls_test.gonew file mode 100644index 000000000..65cd31f4c--- /dev/null+++ b/internal/agent/tools/data_analysis_xls_test.go@@ -0,0 +1,97 @@+package tools++import (+ "context"+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"+ "github.com/xuri/excelize/v2"+)++func TestPrepareExcelForDuckDBConvertsLegacyXLS(t *testing.T) {+ src := filepath.Join("testdata", "legacy.xls")+ converted, cleanup, err := prepareExcelForDuckDB(src)+ require.NoError(t, err)+ t.Cleanup(cleanup)+ require.NotEqual(t, src, converted)++ book, err := excelize.OpenFile(converted)+ require.NoError(t, err)+ t.Cleanup(func() { _ = book.Close() })++ sales, err := book.GetRows("销售")+ require.NoError(t, err)+ require.Equal(t, []string{"名称", "数量"}, sales[0])+ require.Equal(t, []string{"苹果", "3"}, sales[1])+ require.Equal(t, []string{"梨", "12.5"}, sales[2])++ stock, err := book.GetRows("库存")+ require.NoError(t, err)+ require.Equal(t, [][]string{{"sku"}, {"A-1"}}, stock)+}++func TestPrepareExcelForDuckDBLeavesNonOLEXlsAlone(t *testing.T) {+ path := filepath.Join(t.TempDir(), "not.xls")+ require.NoError(t, os.WriteFile(path, []byte("PK\x03\x04not-an-xls"), 0o644))+ got, cleanup, err := prepareExcelForDuckDB(path)+ require.NoError(t, err)+ cleanup()+ require.Equal(t, path, got)+}++func TestPrepareExcelForDuckDBAcceptsUppercaseExtension(t *testing.T) {+ data, err := os.ReadFile(filepath.Join("testdata", "legacy.xls"))+ require.NoError(t, err)+ path := filepath.Join(t.TempDir(), "LEGACY.XLS")+ require.NoError(t, os.WriteFile(path, data, 0o644))++ converted, cleanup, err := prepareExcelForDuckDB(path)+ require.NoError(t, err)+ t.Cleanup(cleanup)++ book, err := excelize.OpenFile(converted)+ require.NoError(t, err)+ t.Cleanup(func() { _ = book.Close() })+ rows, err := book.GetRows("销售")+ require.NoError(t, err)+ require.Equal(t, []string{"苹果", "3"}, rows[1])+}++func TestUniqueExcelSheetNameStripsReservedCharacters(t *testing.T) {+ used := map[string]struct{}{}+ first := uniqueExcelSheetName("a/b:c", 0, used)+ second := uniqueExcelSheetName("a/b:c", 1, used)+ require.Equal(t, "a_b_c", first)+ require.NotEqual(t, first, second)+ require.LessOrEqual(t, len([]rune(second)), 31)+}++func TestPrepareExcelForDuckDBLeavesXLSXAlone(t *testing.T) {+ path := filepath.Join(t.TempDir(), "book.xlsx")+ writeWorkbook(t, path, map[string][][]any{+ "Sheet1": {{"a"}, {"b"}},+ }, []string{"Sheet1"})++ got, cleanup, err := prepareExcelForDuckDB(path)+ require.NoError(t, err)+ cleanup()+ require.Equal(t, path, got)+}++func TestLoadFromExcelReadsLegacyXLS(t *testing.T) {+ db := newTestDuckDB(t)+ tool := &DataAnalysisTool{+ BaseTool: dataAnalysisTool,+ db: db,+ sessionID: "test-legacy-xls",+ }+ ctx := context.Background()+ schema, err := tool.LoadFromExcel(ctx, filepath.Join("testdata", "legacy.xls"), "t_legacy")+ require.NoError(t, err)+ t.Cleanup(func() { tool.Cleanup(ctx) })++ // Two data rows on 销售 and one on 库存.+ require.Equal(t, int64(3), schema.RowCount)+}diff --git a/internal/agent/tools/testdata/legacy.xls b/internal/agent/tools/testdata/legacy.xlsnew file mode 100644index 000000000..0d638c57eBinary files /dev/null and b/internal/agent/tools/testdata/legacy.xls differ