Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix: keep failed data-analysis table loads retryable (#3745)
Perfectoa8c291c6ff52
diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.goindex 561db4d71..36b8dda42 100644--- a/internal/agent/tools/data_analysis.go+++ b/internal/agent/tools/data_analysis.go@@ -129,7 +129,8 @@ type DataAnalysisTool struct {tenantService interfaces.TenantServicedb *sql.DBsessionID string- createdTables []string // Track tables created in this session+ createdTables []string // Cleanup candidates, including failed CREATE attempts+ loadedTables map[string]bool // CREATE succeeded, even if a subsequent schema read failed// localBaseDir is the LOCAL_STORAGE_BASE_DIR value captured at construction// time so resolveFileServiceForKnowledge uses the same base path that was// used when the local FileService was initialised by DI. Re-reading the@@ -182,16 +183,21 @@ func NewDataAnalysisTool(return tool}-// recordCreatedTable records a table name for cleanup, ensuring uniqueness-// Returns true if the table was newly recorded, false if it already existed-func (t *DataAnalysisTool) recordCreatedTable(tableName string) bool {+// recordCreatedTable records a cleanup candidate, not proof that CREATE succeeded.+func (t *DataAnalysisTool) recordCreatedTable(tableName string) {for _, name := range t.createdTables {if name == tableName {- return false+ return}}t.createdTables = append(t.createdTables, tableName)- return true+}++func (t *DataAnalysisTool) markTableLoaded(tableName string) {+ if t.loadedTables == nil {+ t.loadedTables = make(map[string]bool)+ }+ t.loadedTables[tableName] = true}// Cleanup cleans up the session-specific schema@@ -213,8 +219,9 @@ func (t *DataAnalysisTool) Cleanup(ctx context.Context) {logger.Infof(ctx, "[Tool][DataAnalysis] Successfully dropped table '%s'", tableName)}- // Clear the list after cleanup+ // Reset both cleanup tracking and successful-load state.t.createdTables = nil+ t.loadedTables = nil}// Execute executes the SQL query on DuckDB (only read-only queries are allowed)@@ -478,8 +485,10 @@ type ColumnInfo struct {func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tableName string) (*TableSchema, error) {logger.Infof(ctx, "[Tool][DataAnalysis] Loading CSV file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)- // Record the created table for cleanup. If already exists, skip creation- if t.recordCreatedTable(tableName) {+ if !t.loadedTables[tableName] {+ // Keep cleanup tracking even if CREATE fails; only successful creation+ // makes the table reusable by a later call.+ t.recordCreatedTable(tableName)// Create table from CSV using DuckDB's read_csv_auto function// with explicit header detection and VARCHAR coercion to align with// Excel loading behavior.@@ -494,6 +503,7 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tablogger.Errorf(ctx, "[Tool][DataAnalysis] Failed to create table from CSV: %v", err)return nil, fmt.Errorf("failed to create table from CSV: %w", err)}+ t.markTableLoaded(tableName)logger.Infof(ctx, "[Tool][DataAnalysis] Successfully created table '%s' from CSV file in session %s", tableName, t.sessionID)}@@ -524,8 +534,8 @@ func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tabfunc (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) {logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID)- // Record the created table for cleanup. If already exists, skip creation.- if t.recordCreatedTable(tableName) {+ if !t.loadedTables[tableName] {+ t.recordCreatedTable(tableName)sheetNames, enumErr := t.listExcelSheets(ctx, filename)if enumErr != nil {logger.Warnf(ctx,@@ -540,6 +550,7 @@ func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tlogger.Errorf(ctx, "[Tool][DataAnalysis] Failed to create table from Excel (sheets=%v): %v", sheetNames, err)return nil, fmt.Errorf("failed to create table from Excel file (sheets=%v): %w", sheetNames, err)}+ t.markTableLoaded(tableName)logger.Infof(ctx,"[Tool][DataAnalysis] Successfully created table '%s' from Excel file in session %s (sheets=%v)",diff --git a/internal/agent/tools/data_analysis_recovery_test.go b/internal/agent/tools/data_analysis_recovery_test.gonew file mode 100644index 000000000..ff669fb5f--- /dev/null+++ b/internal/agent/tools/data_analysis_recovery_test.go@@ -0,0 +1,216 @@+package tools++import (+ "context"+ "encoding/json"+ "errors"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/DATA-DOG/go-sqlmock"++ "github.com/Tencent/WeKnora/internal/types"+)++func TestDataAnalysisRegistryRetriesFailedCSVLoad(t *testing.T) {+ ctx := context.Background()+ knowledge := &types.Knowledge{ID: "retry-doc", FileType: "csv", FilePath: "retry.csv"}+ files := &csvFileService{files: map[string]string{+ knowledge.FilePath: "id,amount\n1,\xff\n",+ }}+ tool := NewDataAnalysisTool(nil,+ &mapKnowledgeService{docs: map[string]*types.Knowledge{knowledge.ID: knowledge}},+ nil, files, newPlainDuckDB(t), "test-registry-load-retry")+ registry := NewToolRegistry()+ registry.RegisterTool(tool)+ t.Cleanup(func() { tool.Cleanup(ctx) })+ args, err := json.Marshal(DataAnalysisInput{+ KnowledgeID: knowledge.ID,+ SQL: `SELECT COUNT(*) AS total FROM dataset WHERE amount = '20'`,+ })+ if err != nil {+ t.Fatal(err)+ }++ result, err := registry.ExecuteTool(ctx, ToolDataAnalysis, args)+ if err == nil || result == nil || result.Success ||+ !strings.Contains(err.Error(), "failed to create table from CSV") {+ t.Fatalf("first call must reach CREATE and fail: result=%+v err=%v", result, err)+ }++ files.files[knowledge.FilePath] = "id,amount\n1,10\n2,20\n"+ result, err = registry.ExecuteTool(ctx, ToolDataAnalysis, args)+ if err != nil || result == nil || !result.Success {+ t.Fatalf("retry must load the repaired source: result=%+v err=%v", result, err)+ }+ if !strings.Contains(result.Output, `"total":"1"`) {+ t.Fatalf("unexpected query output: %s", result.Output)+ }+}++func TestLoadFromExcelRetriesFailedCreate(t *testing.T) {+ ctx := context.Background()+ tool := &DataAnalysisTool{db: newTestDuckDB(t), sessionID: "test-excel-load-retry"}+ t.Cleanup(func() { tool.Cleanup(ctx) })+ path := filepath.Join(t.TempDir(), "retry.xlsx")++ if _, err := tool.LoadFromExcel(ctx, path, "excel_retry"); err == nil ||+ !strings.Contains(err.Error(), "failed to create table from Excel") {+ t.Fatalf("first call must fail during CREATE: %v", err)+ }+ writeWorkbook(t, path, map[string][][]any{+ "Sales": {{"id", "amount"}, {1, 10}, {2, 20}},+ }, []string{"Sales"})+ schema, err := tool.LoadFromExcel(ctx, path, "excel_retry")+ if err != nil {+ t.Fatalf("retry must load the available workbook: %v", err)+ }+ if schema.RowCount != 2 {+ t.Fatalf("row count = %d, want 2", schema.RowCount)+ }+}++func TestDataAnalysisLoadRecoveryPreservesReuseAndCleanup(t *testing.T) {+ for _, fileType := range []string{"csv", "xlsx"} {+ t.Run(fileType, func(t *testing.T) {+ ctx := context.Background()+ tool := &DataAnalysisTool{sessionID: "test-load-lifecycle"}+ path := filepath.Join(t.TempDir(), "data."+fileType)+ var load func(context.Context, string, string) (*TableSchema, error)+ var write func(bool)+ if fileType == "csv" {+ tool.db = newPlainDuckDB(t)+ load = tool.LoadFromCSV+ write = func(extraRow bool) {+ content := "id,amount\n1,10\n2,20\n"+ if extraRow {+ content += "3,30\n"+ }+ if err := os.WriteFile(path, []byte(content), 0o600); err != nil {+ t.Fatal(err)+ }+ }+ } else {+ tool.db = newTestDuckDB(t)+ load = tool.LoadFromExcel+ write = func(extraRow bool) {+ rows := [][]any{{"id", "amount"}, {1, 10}, {2, 20}}+ if extraRow {+ rows = append(rows, []any{3, 30})+ }+ writeWorkbook(t, path, map[string][][]any{"Sales": rows}, []string{"Sales"})+ }+ }+ t.Cleanup(func() { tool.Cleanup(ctx) })+ write(false)+ canceled, cancel := context.WithCancel(ctx)+ cancel()+ if _, err := load(canceled, path, "recovering"); !errors.Is(err, context.Canceled) {+ t.Fatalf("canceled CREATE must preserve cancellation: %v", err)+ }+ schema, err := load(ctx, path, "recovering")+ if err != nil || schema.RowCount != 2 {+ t.Fatalf("retry after cancellation: schema=%+v err=%v", schema, err)+ }++ // A different table's failed CREATE must not invalidate a ready table.+ if _, err := load(canceled, path, "other"); !errors.Is(err, context.Canceled) {+ t.Fatalf("other table cancellation: %v", err)+ }+ if err := os.Remove(path); err != nil {+ t.Fatal(err)+ }+ if _, err := load(canceled, path, "recovering"); !errors.Is(err, context.Canceled) ||+ !strings.Contains(err.Error(), "failed to get table schema") {+ t.Fatalf("reading a ready table must preserve cancellation: %v", err)+ }+ schema, err = load(ctx, path, "recovering")+ if err != nil || schema.RowCount != 2 {+ t.Fatalf("ready table must be reused with its source missing: schema=%+v err=%v", schema, err)+ }++ tool.Cleanup(ctx)+ if _, err := tool.LoadFromTable(ctx, "recovering"); err == nil {+ t.Fatal("cleanup must remove the loaded table")+ }+ tool.Cleanup(ctx)+ write(true)+ schema, err = load(ctx, path, "recovering")+ if err != nil || schema.RowCount != 3 {+ t.Fatalf("load after cleanup must use fresh data: schema=%+v err=%v", schema, err)+ }+ })+ }+}++func TestDataAnalysisLoadRetainsReadyTableAfterSchemaError(t *testing.T) {+ // Inject a database read failure after a successful CREATE. Real DuckDB+ // tests above cover loading; this seam makes the failure timing deterministic.+ for _, format := range []string{"csv", "xlsx"} {+ t.Run(format, func(t *testing.T) {+ db, mock, err := sqlmock.New()+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ mock.ExpectClose()+ if err := db.Close(); err != nil {+ t.Errorf("close test database: %v", err)+ }+ })+ tool := &DataAnalysisTool{db: db, sessionID: "test-schema-read-retry"}+ load := tool.LoadFromCSV+ path := "source." + format+ if format == "xlsx" {+ load = tool.LoadFromExcel+ mock.ExpectQuery(`SELECT UNNEST\(layers\).name FROM st_read_meta`).WillReturnRows(+ sqlmock.NewRows([]string{"name"}).AddRow("Sales"))+ }+ readErr := errors.New("temporary schema read failure")+ mock.ExpectExec(`CREATE TABLE "ready" AS SELECT`).WillReturnResult(sqlmock.NewResult(0, 2))+ mock.ExpectQuery(`DESCRIBE "ready"`).WillReturnError(readErr)+ if _, err := load(context.Background(), path, "ready"); !errors.Is(err, readErr) {+ t.Fatalf("first load must return the read error: %v", err)+ }+ // No second CREATE (or DROP): the first one already succeeded.+ mock.ExpectQuery(`DESCRIBE "ready"`).WillReturnRows(+ sqlmock.NewRows([]string{"name", "type", "nullable"}).AddRow("id", "VARCHAR", "YES"))+ mock.ExpectQuery(`SELECT COUNT\(\*\) FROM "ready"`).WillReturnRows(+ sqlmock.NewRows([]string{"count"}).AddRow(2))+ schema, err := load(context.Background(), path, "ready")+ if err != nil || schema.RowCount != 2 {+ t.Fatalf("retry should read the existing table: schema=%+v err=%v", schema, err)+ }+ if err := mock.ExpectationsWereMet(); err != nil {+ t.Fatal(err)+ }+ })+ }+}++func TestDataAnalysisCleanupIncludesFailedCreate(t *testing.T) {+ db, mock, err := sqlmock.New()+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ mock.ExpectClose()+ if err := db.Close(); err != nil {+ t.Errorf("close test database: %v", err)+ }+ })+ tool := &DataAnalysisTool{db: db, sessionID: "test-failed-create-cleanup"}+ createErr := errors.New("creation failed")+ mock.ExpectExec(`CREATE TABLE "uncertain" AS SELECT`).WillReturnError(createErr)+ if _, err := tool.LoadFromCSV(context.Background(), "source.csv", "uncertain"); !errors.Is(err, createErr) {+ t.Fatalf("load must preserve the CREATE error: %v", err)+ }+ mock.ExpectExec(`DROP TABLE IF EXISTS "uncertain"`).WillReturnResult(sqlmock.NewResult(0, 0))+ tool.Cleanup(context.Background())+ tool.Cleanup(context.Background())+ if err := mock.ExpectationsWereMet(); err != nil {+ t.Fatal(err)+ }+}