Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(tsql): preserve CTEs in prior backup statements
rebelice46ced781b515
diff --git a/backend/plugin/parser/tsql/backup.go b/backend/plugin/parser/tsql/backup.goindex cffbf886de..45d8e43f4c 100644--- a/backend/plugin/parser/tsql/backup.go+++ b/backend/plugin/parser/tsql/backup.go@@ -120,6 +120,15 @@ func generateSQLForTable(statementInfoList []statementInfo, targetDatabase strintargetTable := fmt.Sprintf("%s_%s_%s", tablePrefix, table.Table, table.Database)targetTable, _ = common.TruncateString(targetTable, maxTableNameLength)var buf strings.Builder+ cteClause, err := backupCTEClause(statementInfoList)+ if err != nil {+ return nil, err+ }+ if cteClause != "" {+ if _, err := fmt.Fprintf(&buf, "%s\n", cteClause); err != nil {+ return nil, errors.Wrap(err, "failed to write buffer")+ }+ }if _, err := fmt.Fprintf(&buf, "SELECT * INTO [%s].[%s].[%s] FROM (\n", targetDatabase, defaultSchema, targetTable); err != nil {return nil, errors.Wrap(err, "failed to write buffer")}@@ -169,6 +178,71 @@ func generateSQLForTable(statementInfoList []statementInfo, targetDatabase strin}, nil}+// backupCTEClause merges the WITH clauses of every statement in the group into+// one clause for the backup SELECT INTO. T-SQL only allows a CTE list at the+// top of a statement, so a CTE referenced inside a UNION branch must be+// hoisted here. The same CTE name across statements must carry the same+// definition, because the merged clause can only bind it once.+func backupCTEClause(statementInfoList []statementInfo) (string, error) {+ var ctes []string+ definitions := make(map[string]string)+ for _, item := range statementInfoList {+ withClause := dmlWithClause(item.node)+ if withClause == nil || withClause.CTEs == nil {+ continue+ }+ if withClause.XmlNamespaces != nil {+ return "", errors.New("prior backup does not support WITH XMLNAMESPACES")+ }+ for _, cteNode := range withClause.CTEs.Items {+ cte, ok := cteNode.(*ast.CommonTableExpr)+ if !ok {+ continue+ }+ definition := sourceFromLoc(item.statement, cte.Loc)+ key := strings.ToLower(cte.Name)+ if previous, ok := definitions[key]; ok {+ if previous != definition {+ return "", errors.Errorf("prior backup cannot handle conflicting definitions of CTE %q across statements", cte.Name)+ }+ continue+ }+ definitions[key] = definition+ ctes = append(ctes, definition)+ }+ }+ if len(ctes) == 0 {+ return "", nil+ }+ return "WITH " + strings.Join(ctes, ",\n"), nil+}++func dmlWithClause(node ast.Node) *ast.WithClause {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ return n.WithClause+ case *ast.DeleteStmt:+ return n.WithClause+ default:+ return nil+ }+}++// isCTETarget reports whether an unqualified DML target names a CTE declared+// by the statement's own WITH clause. Such a DML writes through the CTE to its+// base table, which the backup cannot resolve, so the caller skips it.+func isCTETarget(ref *ast.TableRef, withClause *ast.WithClause) bool {+ if ref == nil || withClause == nil || withClause.CTEs == nil || ref.Database != "" || ref.Schema != "" {+ return false+ }+ for _, cteNode := range withClause.CTEs.Items {+ if cte, ok := cteNode.(*ast.CommonTableExpr); ok && strings.EqualFold(cte.Name, ref.Object) {+ return true+ }+ }+ return false+}+func extractSuffixSelectStatement(node ast.Node, source string) (string, string, error) {switch n := node.(type) {case *ast.UpdateStmt:@@ -202,15 +276,16 @@ func prepareTransformation(databaseName, statement string) ([]statementInfo, err}var (table *TableReference+ targetRef *ast.TableRefstatementType StatementTypeerr error)switch n := node.(type) {case *ast.UpdateStmt:- table, err = resolveDMLTargetTable(n.Relation, n.FromClause, databaseName)+ table, targetRef, err = resolveDMLTargetTable(n.Relation, n.FromClause, databaseName)statementType = StatementTypeUpdatecase *ast.DeleteStmt:- table, err = resolveDMLTargetTable(n.Relation, n.FromClause, databaseName)+ table, targetRef, err = resolveDMLTargetTable(n.Relation, n.FromClause, databaseName)statementType = StatementTypeDeletedefault:continue@@ -227,6 +302,12 @@ func prepareTransformation(databaseName, statement string) ([]statementInfo, err"statementType", statementType)continue}+ if isCTETarget(targetRef, dmlWithClause(node)) {+ slog.Info("prior backup: skipping DML targeting CTE",+ "cte", table.Table,+ "statementType", statementType)+ continue+ }table.StatementType = statementTypeloc := dmlNodeLoc(node)dmls = append(dmls, statementInfo{@@ -241,24 +322,24 @@ func prepareTransformation(databaseName, statement string) ([]statementInfo, errreturn dmls, nil}-func resolveDMLTargetTable(relation ast.TableExpr, fromClause *ast.List, databaseName string) (*TableReference, error) {- table, err := tableReferenceFromTableExpr(relation, databaseName, defaultSchema)- if err != nil || table == nil {- return table, err+// resolveDMLTargetTable returns the DML target and the TableRef it was+// resolved from. When the target is an alias declared in the FROM clause, both+// describe the aliased physical table.+func resolveDMLTargetTable(relation ast.TableExpr, fromClause *ast.List, databaseName string) (*TableReference, *ast.TableRef, error) {+ ref, ok := relation.(*ast.TableRef)+ if !ok {+ return nil, nil, errors.Errorf("unsupported DML target table source %T", relation)}+ table := tableReferenceFromTableRef(ref, databaseName, defaultSchema)if fromClause != nil && table.Database == databaseName && table.Schema == defaultSchema {- if physical := findPhysicalTableForAlias(fromClause, table); physical != nil {- return physical, nil+ if physical, physicalRef := findPhysicalTableForAlias(fromClause, table); physical != nil {+ return physical, physicalRef, nil}}- return table, nil+ return table, ref, nil}-func tableReferenceFromTableExpr(expr ast.TableExpr, defaultDatabase, defaultSchema string) (*TableReference, error) {- ref, ok := expr.(*ast.TableRef)- if !ok {- return nil, errors.Errorf("unsupported DML target table source %T", expr)- }+func tableReferenceFromTableRef(ref *ast.TableRef, defaultDatabase, defaultSchema string) *TableReference {schemaName := defaultSchemaif ref.Schema != "" {schemaName = ref.Schema@@ -272,49 +353,43 @@ func tableReferenceFromTableExpr(expr ast.TableExpr, defaultDatabase, defaultSchSchema: schemaName,Table: ref.Object,Alias: ref.Alias,- }, nil+ }}-func findPhysicalTableForAlias(list *ast.List, table *TableReference) *TableReference {+func findPhysicalTableForAlias(list *ast.List, table *TableReference) (*TableReference, *ast.TableRef) {if list == nil || table == nil {- return nil+ return nil, nil}for _, item := range list.Items {- if result := findPhysicalTableForAliasInNode(item, table); result != nil {- return result+ if result, ref := findPhysicalTableForAliasInNode(item, table); result != nil {+ return result, ref}}- return nil+ return nil, nil}-func findPhysicalTableForAliasInNode(node ast.Node, table *TableReference) *TableReference {+func findPhysicalTableForAliasInNode(node ast.Node, table *TableReference) (*TableReference, *ast.TableRef) {switch n := node.(type) {case *ast.TableRef:if n.Alias != "" && n.Alias == table.Table {- ref, err := tableReferenceFromTableExpr(n, table.Database, table.Schema)- if err != nil {- return nil- }- ref.Alias = n.Alias- return ref+ result := tableReferenceFromTableRef(n, table.Database, table.Schema)+ result.Alias = n.Alias+ return result, n}case *ast.AliasedTableRef:if ref, ok := n.Table.(*ast.TableRef); ok && n.Alias == table.Table {- result, err := tableReferenceFromTableExpr(ref, table.Database, table.Schema)- if err != nil {- return nil- }+ result := tableReferenceFromTableRef(ref, table.Database, table.Schema)result.Alias = n.Alias- return result+ return result, ref}case *ast.JoinClause:- if result := findPhysicalTableForAliasInNode(n.Left, table); result != nil {- return result+ if result, ref := findPhysicalTableForAliasInNode(n.Left, table); result != nil {+ return result, ref}return findPhysicalTableForAliasInNode(n.Right, table)default:}- return nil+ return nil, nil}func dmlFromSource(source string, relation ast.TableExpr, fromClause *ast.List, stmtLoc ast.Loc, where ast.ExprNode, option *ast.List) (string, int) {diff --git a/backend/plugin/parser/tsql/backup_test.go b/backend/plugin/parser/tsql/backup_test.goindex d2588d96c4..f4f79fa2a0 100644--- a/backend/plugin/parser/tsql/backup_test.go+++ b/backend/plugin/parser/tsql/backup_test.go@@ -87,6 +87,62 @@ func TestBackupOmniBoundaryCases(t *testing.T) {input: "UPDATE test SET c1 = 1 WHERE CURRENT OF my_cursor;",wantErrPart: "CURSOR clause is not supported",},+ {+ name: "update with cte",+ input: "WITH c AS (SELECT id FROM src) UPDATE dbo.test SET c1 = 1 WHERE EXISTS (SELECT 1 FROM c WHERE c.id = test.id);",+ wantSQL: strings.Join([]string{+ "WITH c AS (SELECT id FROM src)",+ "SELECT * INTO [backupDB].[dbo].[rollback_test_db] FROM (",+ " SELECT [db].[dbo].[test].* FROM dbo.test WHERE EXISTS (SELECT 1 FROM c WHERE c.id = test.id)) AS backup_table;",+ }, "\n"),+ },+ {+ name: "delete with cte",+ input: "WITH c (id) AS (SELECT id FROM src), d AS (SELECT 1 AS n) DELETE FROM test WHERE id IN (SELECT id FROM c);",+ wantSQL: strings.Join([]string{+ "WITH c (id) AS (SELECT id FROM src),",+ "d AS (SELECT 1 AS n)",+ "SELECT * INTO [backupDB].[dbo].[rollback_test_db] FROM (",+ " SELECT [db].[dbo].[test].* FROM test WHERE id IN (SELECT id FROM c)) AS backup_table;",+ }, "\n"),+ },+ {+ name: "update joined to cte",+ input: "WITH c AS (SELECT id FROM src) UPDATE t SET c1 = 1 FROM test AS t JOIN c ON t.id = c.id;",+ wantSQL: strings.Join([]string{+ "WITH c AS (SELECT id FROM src)",+ "SELECT * INTO [backupDB].[dbo].[rollback_test_db] FROM (",+ " SELECT [t].* FROM test AS t JOIN c ON t.id = c.id) AS backup_table;",+ }, "\n"),+ },+ {+ name: "cte lists merge across statements",+ input: strings.Join([]string{+ "WITH c AS (SELECT id FROM src) UPDATE test SET c1 = 1 WHERE id IN (SELECT id FROM c);",+ "WITH c AS (SELECT id FROM src), d AS (SELECT id FROM other) UPDATE test SET c1 = 2 WHERE id IN (SELECT id FROM d);",+ }, "\n"),+ wantSQL: strings.Join([]string{+ "WITH c AS (SELECT id FROM src),",+ "d AS (SELECT id FROM other)",+ "SELECT * INTO [backupDB].[dbo].[rollback_test_db] FROM (",+ " SELECT [db].[dbo].[test].* FROM test WHERE id IN (SELECT id FROM c)",+ " UNION",+ " SELECT [db].[dbo].[test].* FROM test WHERE id IN (SELECT id FROM d)) AS backup_table;",+ }, "\n"),+ },+ {+ name: "conflicting cte definitions rejected",+ input: strings.Join([]string{+ "WITH c AS (SELECT id FROM src) UPDATE test SET c1 = 1 WHERE id IN (SELECT id FROM c);",+ "WITH c AS (SELECT id FROM other) UPDATE test SET c1 = 2 WHERE id IN (SELECT id FROM c);",+ }, "\n"),+ wantErrPart: `conflicting definitions of CTE "c"`,+ },+ {+ name: "xmlnamespaces rejected",+ input: "WITH XMLNAMESPACES ('uri' AS ns), c AS (SELECT id FROM src) UPDATE test SET c1 = 1 WHERE id IN (SELECT id FROM c);",+ wantErrPart: "WITH XMLNAMESPACES",+ },}for _, tc := range tests {@@ -104,6 +160,37 @@ func TestBackupOmniBoundaryCases(t *testing.T) {}}+func TestBackupSkipsCTETarget(t *testing.T) {+ for name, input := range map[string]string{+ "direct": "WITH c AS (SELECT id, c1 FROM test) UPDATE c SET c1 = 1 WHERE id = 1;",+ "alias": "WITH c AS (SELECT id, c1 FROM test) UPDATE x SET c1 = 1 FROM c AS x WHERE x.id = 1;",+ "delete": "WITH c AS (SELECT id FROM test) DELETE FROM c WHERE id = 1;",+ "mixed case": "WITH C AS (SELECT id, c1 FROM test) UPDATE c SET c1 = 1 WHERE id = 1;",+ "other backed": "WITH c AS (SELECT id FROM test) UPDATE c SET c1 = 1 WHERE id = 1; DELETE FROM test WHERE id = 2;",+ } {+ t.Run(name, func(t *testing.T) {+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{}, input, "db", "backupDB", "rollback")+ require.NoError(t, err)+ for _, item := range result {+ require.NotEqual(t, "c", item.SourceTableName, item.Statement)+ require.NotContains(t, item.Statement, "WITH", item.Statement)+ }+ if name == "other backed" {+ require.Len(t, result, 1)+ require.Equal(t, "test", result[0].SourceTableName)+ } else {+ require.Empty(t, result)+ }+ })+ }+ // A schema-qualified target is a real table even when a CTE shares its name.+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{}, "WITH c AS (SELECT id FROM src) UPDATE dbo.c SET c1 = 1 WHERE id IN (SELECT id FROM c);", "db", "backupDB", "rollback")+ require.NoError(t, err)+ require.Len(t, result, 1)+ require.Equal(t, "c", result[0].SourceTableName)+ require.True(t, strings.HasPrefix(result[0].Statement, "WITH c AS (SELECT id FROM src)\n"), result[0].Statement)+}+func TestBackup(t *testing.T) {tests := []rollbackCase{}