Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(pg): reject prior backup for data-modifying WITH queries (#21583)
rebelice3ec211eb5900
diff --git a/backend/plugin/advisor/pg/advisor_builtin_prior_backup_check.go b/backend/plugin/advisor/pg/advisor_builtin_prior_backup_check.goindex 7a2851b3a8..ea9550934f 100644--- a/backend/plugin/advisor/pg/advisor_builtin_prior_backup_check.go+++ b/backend/plugin/advisor/pg/advisor_builtin_prior_backup_check.go@@ -3,6 +3,7 @@ package pgimport ("context""fmt"+ "strings""github.com/bytebase/omni/pg/ast"@@ -68,6 +69,18 @@ func (*BuiltinPriorBackupCheckAdvisor) Check(_ context.Context, checkCtx advisor},})}+ if content := dataModifyingCTEAdvice(node); content != "" {+ adviceList = append(adviceList, &storepb.Advice{+ Status: level,+ Title: title,+ Content: content,+ Code: code.BuiltinPriorBackupCheck.Int32(),+ StartPosition: &storepb.Position{+ Line: int32(baseLine) + omniContentStartLine(stmt.Text),+ Column: 0,+ },+ })+ }}// Check if backup schema exists@@ -114,6 +127,27 @@ func (*BuiltinPriorBackupCheckAdvisor) Check(_ context.Context, checkCtx advisorreturn adviceList, nil}+// dataModifyingCTEAdvice reports a statement whose WITH clause changes data.+// The backup task rejects such an UPDATE or DELETE; any other statement runs,+// but nothing backs up the rows its CTE changes.+func dataModifyingCTEAdvice(node ast.Node) string {+ names := pgparser.DataModifyingCTENames(node)+ if len(names) == 0 {+ return ""+ }+ quoted := make([]string, 0, len(names))+ for _, name := range names {+ quoted = append(quoted, fmt.Sprintf("%q", name))+ }+ cteList := strings.Join(quoted, ", ")+ switch node.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ return fmt.Sprintf("Prior backup does not support data-modifying WITH query %s; the task will fail unless prior backup is disabled or skipped", cteList)+ default:+ return fmt.Sprintf("Prior backup does not cover rows changed by data-modifying WITH query %s", cteList)+ }+}+// omniIsDDLStatement checks if a node is a DDL statement.func omniIsDDLStatement(node ast.Node) bool {switch node.(type) {diff --git a/backend/plugin/advisor/pg/test/builtin_prior_backup_check.yaml b/backend/plugin/advisor/pg/test/builtin_prior_backup_check.yamlindex 005fd6b619..39ba787669 100644--- a/backend/plugin/advisor/pg/test/builtin_prior_backup_check.yaml+++ b/backend/plugin/advisor/pg/test/builtin_prior_backup_check.yaml@@ -41,3 +41,52 @@endposition: null- statement: SET ROLE role1;DELETE FROM tech_book WHERE a > 1;changeType: 2+- statement: WITH cte AS (SELECT id FROM orders) UPDATE tech_book SET name = 'x' FROM cte WHERE tech_book.id = cte.id;+ changeType: 2+- statement: |-+ UPDATE orders SET id = 1 WHERE id = 1;+ WITH d AS (DELETE FROM orders WHERE id > 1 RETURNING id)+ UPDATE tech_book SET name = 'x' WHERE id IN (SELECT id FROM d);+ changeType: 2+ want:+ - status: 2+ code: 2001+ title: BUILTIN_PRIOR_BACKUP_CHECK+ content: Prior backup does not support data-modifying WITH query "d"; the task will fail unless prior backup is disabled or skipped+ startposition:+ line: 2+ column: 0+ endposition: null+- statement: WITH s AS (SELECT 1), i AS (INSERT INTO orders VALUES (1) RETURNING id), u AS (UPDATE orders SET id = 2 RETURNING id) DELETE FROM tech_book WHERE id IN (SELECT id FROM i);+ changeType: 2+ want:+ - status: 2+ code: 2001+ title: BUILTIN_PRIOR_BACKUP_CHECK+ content: Prior backup does not support data-modifying WITH query "i", "u"; the task will fail unless prior backup is disabled or skipped+ startposition:+ line: 1+ column: 0+ endposition: null+- statement: WITH d AS (DELETE FROM orders RETURNING id) INSERT INTO tech_book (id) SELECT id FROM d;+ changeType: 2+ want:+ - status: 2+ code: 2001+ title: BUILTIN_PRIOR_BACKUP_CHECK+ content: Prior backup does not cover rows changed by data-modifying WITH query "d"+ startposition:+ line: 1+ column: 0+ endposition: null+- statement: WITH d AS (DELETE FROM orders RETURNING id) SELECT id FROM d;+ changeType: 2+ want:+ - status: 2+ code: 2001+ title: BUILTIN_PRIOR_BACKUP_CHECK+ content: Prior backup does not cover rows changed by data-modifying WITH query "d"+ startposition:+ line: 1+ column: 0+ endposition: nulldiff --git a/backend/plugin/parser/pg/backup.go b/backend/plugin/parser/pg/backup.goindex 17000b56e0..333f76b4bc 100644--- a/backend/plugin/parser/pg/backup.go+++ b/backend/plugin/parser/pg/backup.go@@ -179,6 +179,55 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,}, nil}+// DataModifyingCTENames returns the names of the CTEs in a statement's WITH+// clause that run INSERT, UPDATE, DELETE, or MERGE. PostgreSQL only allows+// such CTEs in the top-level WITH clause, so nested queries are not searched.+func DataModifyingCTENames(node ast.Node) []string {+ var wc *ast.WithClause+ switch n := node.(type) {+ case *ast.SelectStmt:+ wc = n.WithClause+ case *ast.InsertStmt:+ wc = n.WithClause+ case *ast.UpdateStmt:+ wc = n.WithClause+ case *ast.DeleteStmt:+ wc = n.WithClause+ case *ast.MergeStmt:+ wc = n.WithClause+ default:+ }+ if wc == nil || wc.Ctes == nil {+ return nil+ }+ var names []string+ for _, item := range wc.Ctes.Items {+ cte, ok := item.(*ast.CommonTableExpr)+ if !ok {+ continue+ }+ if _, isSelect := cte.Ctequery.(*ast.SelectStmt); !isSelect {+ names = append(names, cte.Ctename)+ }+ }+ return names+}++// rejectDataModifyingCTE fails a backup whose statement has a data-modifying+// CTE: the backup copies the WITH clause, so running it would apply that+// change once before the statement itself runs.+func rejectDataModifyingCTE(node ast.Node) error {+ names := DataModifyingCTENames(node)+ if len(names) == 0 {+ return nil+ }+ quoted := make([]string, 0, len(names))+ for _, name := range names {+ quoted = append(quoted, fmt.Sprintf("%q", name))+ }+ return errors.Errorf("prior backup does not support data-modifying WITH query %s; rewrite the statement without it, or disable or skip prior backup", strings.Join(quoted, ", "))+}+func extractCTE(node ast.Node, fullSQL string) string {var wc *ast.WithClauseswitch n := node.(type) {@@ -317,6 +366,9 @@ func prepareTransformation(ctx context.Context, tCtx base.TransformContext, stat}}case *ast.UpdateStmt:+ if err := rejectDataModifyingCTE(n); err != nil {+ return nil, err+ }table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)if err != nil {return nil, errors.Wrapf(err, "failed to extract table reference from update statement at offset %d", i)@@ -339,6 +391,9 @@ func prepareTransformation(ctx context.Context, tCtx base.TransformContext, statfullSQL: statement,})case *ast.DeleteStmt:+ if err := rejectDataModifyingCTE(n); err != nil {+ return nil, err+ }table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)if err != nil {return nil, errors.Wrapf(err, "failed to extract table reference from delete statement at offset %d", i)diff --git a/backend/plugin/parser/pg/backup_test.go b/backend/plugin/parser/pg/backup_test.goindex ec64cf3df1..feb3bc6774 100644--- a/backend/plugin/parser/pg/backup_test.go+++ b/backend/plugin/parser/pg/backup_test.go@@ -84,3 +84,60 @@ func TestBackupUnsyncedDatabase(t *testing.T) {}, "UPDATE t SET a = 1 WHERE b = 1;", "db", "backupSchema", "rollback")require.ErrorContains(t, err, `database metadata for database "db" not found (database not synced)`)}++func TestBackupRejectsDataModifyingCTE(t *testing.T) {+ tests := []struct {+ name string+ statement string+ wantErr string+ }{+ {+ name: "delete CTE feeding update",+ statement: "WITH d AS (DELETE FROM test2 WHERE a > 1 RETURNING a) UPDATE test SET c = 1 WHERE a IN (SELECT a FROM d);",+ wantErr: `data-modifying WITH query "d";`,+ },+ {+ name: "insert CTE feeding delete",+ statement: "WITH i AS (INSERT INTO test2 (a) VALUES (1) RETURNING a) DELETE FROM test USING i WHERE test.a = i.a;",+ wantErr: `data-modifying WITH query "i";`,+ },+ {+ name: "update CTE beside read-only CTE",+ statement: "WITH s AS (SELECT a FROM test2), u AS (UPDATE test2 SET a = a + 1 RETURNING a) UPDATE test SET c = 1 FROM s WHERE test.a = s.a;",+ wantErr: `data-modifying WITH query "u";`,+ },+ {+ name: "merge CTE",+ statement: "WITH m AS (MERGE INTO test2 USING test ON test2.a = test.a WHEN MATCHED THEN DELETE RETURNING test2.a) DELETE FROM test WHERE a IN (SELECT a FROM m);",+ wantErr: `data-modifying WITH query "m";`,+ },+ {+ name: "rejected after an earlier statement transforms",+ statement: "UPDATE test2 SET a = 1;\nWITH d AS (DELETE FROM test2 RETURNING a) DELETE FROM test WHERE a IN (SELECT a FROM d);",+ wantErr: `data-modifying WITH query "d";`,+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, tc.statement, "", "backupSchema", "rollback")+ require.ErrorContains(t, err, tc.wantErr)+ require.Nil(t, result)+ })+ }+}++func TestBackupAllowsReadOnlyCTE(t *testing.T) {+ for _, statement := range []string{+ "WITH v(a) AS (VALUES (1)) UPDATE test SET c = 1 FROM v WHERE test.a = v.a;",+ "WITH s AS (TABLE test2) DELETE FROM test USING s WHERE test.a = s.a;",+ "WITH s AS ((SELECT a FROM test2) UNION SELECT 1) UPDATE test SET c = 1 FROM s WHERE test.a = s.a;",+ } {+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, statement, "", "backupSchema", "rollback")+ require.NoError(t, err, statement)+ require.Len(t, result, 1, statement)+ }+}