Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(tools): reject legacy binary .xls with an actionable error (#4374)
Jay0130-ae07a190219a9
diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.goindex fab508a39..374847d81 100644--- a/internal/agent/tools/data_analysis.go+++ b/internal/agent/tools/data_analysis.go@@ -1,6 +1,7 @@package toolsimport (+ "bytes""context""database/sql""encoding/json"@@ -36,8 +37,9 @@ var dataAnalysisTool = BaseTool{"the single table \"" + DataAnalysisTableName + "\"; write SQL against that table name and never put " +"the document ID inside the SQL. For Excel files with multiple sheets, non-empty sheets are loaded into " +"the same table and the source sheet name is exposed as a '__sheet_name' column so you can " +- "filter/aggregate per sheet. If the user's question requires data statistics, convert the " +- "question into SQL and execute it.",+ "filter/aggregate per sheet. Only the .xlsx container is readable: a legacy binary .xls must be " ++ "re-saved as .xlsx (or exported as .csv) before it can be analysed. If the user's question requires " ++ "data statistics, convert the question into SQL and execute it.",schema: utils.GenerateSchema[DataAnalysisInput](),}@@ -45,6 +47,48 @@ var dataAnalysisTool = BaseTool{// which Excel sheet a row came from when multiple sheets are unioned together.const excelSheetNameColumn = "__sheet_name"+// legacyXLSMagic is the OLE2/CFB compound-document signature that binary BIFF+// .xls workbooks start with ("D0 CF 11 E0 A1 B1 1A E1").+//+// DuckDB's excel extension only reads the ZIP-based OOXML container+// (.xlsx/.xlsm), so a genuine .xls has to be rejected up front: handed to+// read_xlsx it fails with the opaque "No xl/workbook.xml found in xlsx file",+// which says nothing about the real cause or what to do about it.+var legacyXLSMagic = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}++// hasLegacyXLSContainer reports whether path holds an OLE2 compound document,+// i.e. a binary BIFF .xls workbook.+//+// The sniff is by content, not by extension: a .xls that is really a renamed+// .xlsx is a ZIP archive and returns false, so it keeps loading through the+// normal read_xlsx path. Unreadable or truncated files also return false and+// are left to DuckDB to report.+func hasLegacyXLSContainer(path string) bool {+ f, err := os.Open(path)+ if err != nil {+ return false+ }+ defer func() { _ = f.Close() }()++ head := make([]byte, len(legacyXLSMagic))+ if _, err := io.ReadFull(f, head); err != nil {+ return false+ }+ return bytes.Equal(head, legacyXLSMagic)+}++// errLegacyBinaryXLS is the actionable failure returned for a binary .xls+// workbook. The data analysis tool cannot read it, and silently reporting an+// empty table (or leaking DuckDB's xlsx-internals error) leaves the model+// guessing; naming the conversion to perform is what the caller can act on.+func errLegacyBinaryXLS() error {+ return fmt.Errorf(+ "legacy binary .xls workbooks are not supported by data analysis: " ++ "only the ZIP-based .xlsx format can be read; " ++ "re-save the workbook as .xlsx (or export the sheet as .csv) and upload it again",+ )+}+// sqlSingleQuoteEscape escapes single quotes in a string so it can be safely// embedded inside a single-quoted SQL literal.func sqlSingleQuoteEscape(s string) string {@@ -547,9 +591,23 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tab//// Note: requires the DuckDB 'excel' extension (for read_xlsx) and the// 'spatial' extension (for st_read_meta used to enumerate sheets).+//+// Binary BIFF .xls workbooks are rejected with an actionable error before+// DuckDB is touched: the excel extension only reads the ZIP-based OOXML+// container, so a real .xls would otherwise surface as "No xl/workbook.xml+// found in xlsx file".func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) {logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)+ if hasLegacyXLSContainer(filename) {+ logger.Warnf(ctx,+ "[Tool][DataAnalysis] Rejecting legacy binary .xls workbook '%s' (session=%s): "++ "DuckDB can only read the ZIP-based .xlsx container",+ filename, t.sessionID,+ )+ return nil, errLegacyBinaryXLS()+ }+if !t.loadedTables[tableName] {t.recordCreatedTable(tableName)sheetNames, enumErr := t.listExcelSheets(ctx, filename)diff --git a/internal/agent/tools/data_analysis_legacy_xls_test.go b/internal/agent/tools/data_analysis_legacy_xls_test.gonew file mode 100644index 000000000..3868d0dc8--- /dev/null+++ b/internal/agent/tools/data_analysis_legacy_xls_test.go@@ -0,0 +1,91 @@+package tools++import (+ "context"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// ole2Header is the compound-document signature a binary BIFF .xls starts with.+var ole2Header = []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1}++// writeLegacyXLS writes a file that looks like a binary BIFF .xls: the OLE2+// signature followed by opaque payload bytes. DuckDB's read_xlsx cannot open+// this (there is no xl/workbook.xml inside), which is exactly the case #4239+// reports.+func writeLegacyXLS(t *testing.T, path string) {+ t.Helper()+ payload := append([]byte{}, ole2Header...)+ payload = append(payload, make([]byte, 512)...)+ if err := os.WriteFile(path, payload, 0o600); err != nil {+ t.Fatalf("write legacy xls fixture: %v", err)+ }+}++func TestHasLegacyXLSContainerDetectsOLE2Workbook(t *testing.T) {+ path := filepath.Join(t.TempDir(), "legacy.xls")+ writeLegacyXLS(t, path)++ if !hasLegacyXLSContainer(path) {+ t.Fatalf("expected %s to be detected as a binary BIFF workbook", path)+ }+}++func TestHasLegacyXLSContainerIgnoresZipBasedWorkbooks(t *testing.T) {+ path := filepath.Join(t.TempDir(), "modern.xlsx")+ writeWorkbook(t, path, map[string][][]any{+ "Data": {{"id", "amount"}, {1, 100}},+ }, []string{"Data"})++ // A .xls that is really a renamed .xlsx is a ZIP archive and must keep+ // loading through the normal read_xlsx path.+ if hasLegacyXLSContainer(path) {+ t.Fatalf("expected the ZIP-based workbook %s not to be flagged as legacy", path)+ }+}++func TestHasLegacyXLSContainerLeavesShortAndMissingFilesAlone(t *testing.T) {+ dir := t.TempDir()++ empty := filepath.Join(dir, "empty.xls")+ if err := os.WriteFile(empty, nil, 0o600); err != nil {+ t.Fatalf("write empty fixture: %v", err)+ }+ if hasLegacyXLSContainer(empty) {+ t.Fatalf("an empty file is not a legacy workbook")+ }++ if hasLegacyXLSContainer(filepath.Join(dir, "missing.xls")) {+ t.Fatalf("a missing file must not be reported as a legacy workbook")+ }+}++func TestLoadFromExcelRejectsLegacyBinaryXLSWithActionableError(t *testing.T) {+ path := filepath.Join(t.TempDir(), "legacy.xls")+ writeLegacyXLS(t, path)++ // No DuckDB handle: the rejection has to happen before the tool touches+ // the database, otherwise a nil db would panic instead of failing cleanly.+ tool := &DataAnalysisTool{sessionID: "legacy-xls"}++ _, err := tool.LoadFromExcel(context.Background(), path, "dataset")+ if err == nil {+ t.Fatal("expected a binary .xls workbook to be rejected")+ }++ message := err.Error()+ for _, want := range []string{"legacy binary .xls", ".xlsx", ".csv"} {+ if !strings.Contains(message, want) {+ t.Errorf("error %q does not mention %q", message, want)+ }+ }+}++func TestDataAnalysisToolDescriptionNamesTheXLSConversion(t *testing.T) {+ description := dataAnalysisTool.description+ if !strings.Contains(description, ".xlsx") || !strings.Contains(description, "binary .xls") {+ t.Fatalf("tool description must tell the model how to handle a legacy .xls, got: %s", description)+ }+}