Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(agent): 保留数据分析 SQL 字符串字面量 (#3812)

诉说晚秋f6119ae5fd40
diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go
index 36b8dda42..394479471 100644
--- a/internal/agent/tools/data_analysis.go
+++ b/internal/agent/tools/data_analysis.go
@@ -16,6 +16,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
+ pg_query "github.com/pganalyze/pg_query_go/v6"
)
// DataAnalysisTableName is the only table name the model needs to know when
@@ -72,19 +73,33 @@ func reconcileSQLColumnsWithSchema(sqlText string, schema *TableSchema) (string,
}
}
- quotedIdentifierPattern := regexp.MustCompile(`"([^"]+)"`)
+ // Use the same SQL lexer as validation so quoted text in literals and
+ // comments is never mistaken for an identifier. Leave invalid SQL to validation.
+ tokens, err := pg_query.Scan(sqlText)
+ if err != nil {
+ return sqlText, nil
+ }
fixes := make([]string, 0)
- rewritten := quotedIdentifierPattern.ReplaceAllStringFunc(sqlText, func(token string) string {
- name := strings.Trim(token, "\"")
+ var rewritten strings.Builder
+ previous := 0
+ for _, token := range tokens.Tokens {
+ raw := sqlText[token.Start:token.End]
+ if token.Token != pg_query.Token_IDENT || !strings.HasPrefix(raw, `"`) {
+ continue
+ }
+ name := strings.ReplaceAll(raw[1:len(raw)-1], `""`, `"`)
canonical, ok := normalizedToCanonical[normalizeIdentifierForMatch(name)]
if !ok || canonical == name {
- return token
+ continue
}
fixes = append(fixes, fmt.Sprintf("%q -> %q", name, canonical))
- return fmt.Sprintf(`"%s"`, canonical)
- })
+ rewritten.WriteString(sqlText[previous:token.Start])
+ fmt.Fprintf(&rewritten, `"%s"`, strings.ReplaceAll(canonical, `"`, `""`))
+ previous = int(token.End)
+ }
+ rewritten.WriteString(sqlText[previous:])
- return rewritten, fixes
+ return rewritten.String(), fixes
}
func buildMissingColumnSuggestion(sqlErr error, schema *TableSchema) string {
diff --git a/internal/agent/tools/data_analysis_reconcile_test.go b/internal/agent/tools/data_analysis_reconcile_test.go
new file mode 100644
index 000000000..c1dbbe58a
--- /dev/null
+++ b/internal/agent/tools/data_analysis_reconcile_test.go
@@ -0,0 +1,76 @@
+package tools
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestReconcileSQLPreservesNonIdentifiers(t *testing.T) {
+ schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}}
+ for _, literal := range []string{
+ `'"orderstatus"'`,
+ `'customer''s "orderstatus"'`,
+ `$$"orderstatus"$$`,
+ `$value$"orderstatus"$value$`,
+ `E'customer\'s "orderstatus"'`,
+ } {
+ t.Run(literal, func(t *testing.T) {
+ query := `SELECT "orderstatus" FROM dataset WHERE "orderstatus" = ` + literal
+ want := `SELECT "Order Status" FROM dataset WHERE "Order Status" = ` + literal
+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)
+ require.Equal(t, want, got)
+ require.Len(t, fixes, 2)
+ })
+ }
+ for _, comment := range []string{
+ "-- \"orderstatus\"\n",
+ `/* "orderstatus" */`,
+ `/* outer /* "orderstatus" */ end */`,
+ } {
+ t.Run(comment, func(t *testing.T) {
+ query := `SELECT ` + comment + ` "orderstatus" FROM dataset`
+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)
+ require.Equal(t, `SELECT `+comment+` "Order Status" FROM dataset`, got)
+ require.Len(t, fixes, 1)
+ })
+ }
+}
+
+func TestReconcileSQLPreservesQuotedIdentifierEscapes(t *testing.T) {
+ schema := &TableSchema{Columns: []ColumnInfo{{Name: `Order "Status"`}, {Name: "订单 状态"}}}
+ query := `SELECT "order""status""","订单状态" FROM dataset`
+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)
+ require.Equal(t, `SELECT "Order ""Status""","订单 状态" FROM dataset`, got)
+ require.Len(t, fixes, 2)
+}
+
+func TestReconcileSQLLeavesUnscannableInputUnchanged(t *testing.T) {
+ schema := &TableSchema{Columns: []ColumnInfo{{Name: "Order Status"}}}
+ for _, query := range []string{
+ `SELECT "orderstatus" FROM dataset WHERE note = 'unfinished`,
+ `SELECT "orderstatus`,
+ } {
+ got, fixes := reconcileSQLColumnsWithSchema(query, schema)
+ require.Equal(t, query, got)
+ require.Empty(t, fixes)
+ }
+}
+
+func TestReconcileSQLPreservesDuckDBFilterResults(t *testing.T) {
+ db := newPlainDuckDB(t)
+ tool := &DataAnalysisTool{BaseTool: dataAnalysisTool, db: db, sessionID: "literal-filter"}
+ ctx := context.Background()
+ t.Cleanup(func() { tool.Cleanup(ctx) })
+ path := writeCSV(t, "orders.csv", "Order Status,note",
+ `paid,"""orderstatus"""`, `unpaid,"""Order Status"""`)
+ schema, err := tool.LoadFromCSV(ctx, path, "k_literal_filter")
+ require.NoError(t, err)
+ query := `SELECT "orderstatus" AS status FROM dataset WHERE note = '"orderstatus"'`
+ rewritten, _ := reconcileSQLColumnsWithSchema(query, schema)
+ require.NoError(t, validateDataAnalysisSQL(rewritten, schema))
+ rows, err := tool.executeSingleQuery(ctx, rewritten, schema.TableName)
+ require.NoError(t, err)
+ require.Equal(t, []map[string]string{{"status": "paid"}}, rows)
+}