Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(agent): 保留数据分析 SQL 字符串字面量 (#3812)
诉说晚秋f6119ae5fd40
diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.goindex 36b8dda42..394479471 100644--- a/internal/agent/tools/data_analysis.go+++ b/internal/agent/tools/data_analysis.go@@ -16,6 +16,7 @@ import ("github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces""github.com/Tencent/WeKnora/internal/utils"+ pg_query "github.com/pganalyze/pg_query_go/v6")// DataAnalysisTableName is the only table name the model needs to know when@@ -72,19 +73,33 @@ func reconcileSQLColumnsWithSchema(sqlText string, schema *TableSchema) (string,}}- quotedIdentifierPattern := regexp.MustCompile(`"([^"]+)"`)+ // Use the same SQL lexer as validation so quoted text in literals and+ // comments is never mistaken for an identifier. Leave invalid SQL to validation.+ tokens, err := pg_query.Scan(sqlText)+ if err != nil {+ return sqlText, nil+ }fixes := make([]string, 0)- rewritten := quotedIdentifierPattern.ReplaceAllStringFunc(sqlText, func(token string) string {- name := strings.Trim(token, "\"")+ var rewritten strings.Builder+ previous := 0+ for _, token := range tokens.Tokens {+ raw := sqlText[token.Start:token.End]+ if token.Token != pg_query.Token_IDENT || !strings.HasPrefix(raw, `"`) {+ continue+ }+ name := strings.ReplaceAll(raw[1:len(raw)-1], `""`, `"`)canonical, ok := normalizedToCanonical[normalizeIdentifierForMatch(name)]if !ok || canonical == name {- return token+ continue}fixes = append(fixes, fmt.Sprintf("%q -> %q", name, canonical))- return fmt.Sprintf(`"%s"`, canonical)- })+ rewritten.WriteString(sqlText[previous:token.Start])+ fmt.Fprintf(&rewritten, `"%s"`, strings.ReplaceAll(canonical, `"`, `""`))+ previous = int(token.End)+ }+ rewritten.WriteString(sqlText[previous:])- return rewritten, fixes+ return rewritten.String(), fixes}func buildMissingColumnSuggestion(sqlErr error, schema *TableSchema) string {diff --git a/internal/agent/tools/data_analysis_reconcile_test.go b/internal/agent/tools/data_analysis_reconcile_test.gonew file mode 100644index 000000000..c1dbbe58a--- /dev/null+++ b/internal/agent/tools/data_analysis_reconcile_test.go@@ -0,0 +1,76 @@+package tools++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestReconcileSQLPreservesNonIdentifiers(t *testing.T) {+ schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}}+ for _, literal := range []string{+ `'"orderstatus"'`,+ `'customer''s "orderstatus"'`,+ `$$"orderstatus"$$`,+ `$value$"orderstatus"$value$`,+ `E'customer\'s "orderstatus"'`,+ } {+ t.Run(literal, func(t *testing.T) {+ query := `SELECT "orderstatus" FROM dataset WHERE "orderstatus" = ` + literal+ want := `SELECT "Order Status" FROM dataset WHERE "Order Status" = ` + literal+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)+ require.Equal(t, want, got)+ require.Len(t, fixes, 2)+ })+ }+ for _, comment := range []string{+ "-- \"orderstatus\"\n",+ `/* "orderstatus" */`,+ `/* outer /* "orderstatus" */ end */`,+ } {+ t.Run(comment, func(t *testing.T) {+ query := `SELECT ` + comment + ` "orderstatus" FROM dataset`+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)+ require.Equal(t, `SELECT `+comment+` "Order Status" FROM dataset`, got)+ require.Len(t, fixes, 1)+ })+ }+}++func TestReconcileSQLPreservesQuotedIdentifierEscapes(t *testing.T) {+ schema := &TableSchema{Columns: []ColumnInfo{{Name: `Order "Status"`}, {Name: "订单 状态"}}}+ query := `SELECT "order""status""","订单状态" FROM dataset`+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)+ require.Equal(t, `SELECT "Order ""Status""","订单 状态" FROM dataset`, got)+ require.Len(t, fixes, 2)+}++func TestReconcileSQLLeavesUnscannableInputUnchanged(t *testing.T) {+ schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}}+ for _, query := range []string{+ `SELECT "orderstatus" FROM dataset WHERE note = 'unfinished`,+ `SELECT "orderstatus`,+ } {+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)+ require.Equal(t, query, got)+ require.Empty(t, fixes)+ }+}++func TestReconcileSQLPreservesDuckDBFilterResults(t *testing.T) {+ db := newPlainDuckDB(t)+ tool := &DataAnalysisTool{BaseTool: dataAnalysisTool, db: db, sessionID: "literal-filter"}+ ctx := context.Background()+ t.Cleanup(func() { tool.Cleanup(ctx) })+ path := writeCSV(t, "orders.csv", "Order Status,note",+ `paid,"""orderstatus"""`, `unpaid,"""Order Status"""`)+ schema, err := tool.LoadFromCSV(ctx, path, "k_literal_filter")+ require.NoError(t, err)+ query := `SELECT "orderstatus" AS status FROM dataset WHERE note = '"orderstatus"'`+ rewritten, _ := reconcileSQLColumnsWithSchema(query, schema)+ require.NoError(t, validateDataAnalysisSQL(rewritten, schema))+ rows, err := tool.executeSingleQuery(ctx, rewritten, schema.TableName)+ require.NoError(t, err)+ require.Equal(t, []map[string]string{{"status": "paid"}}, rows)+}