Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: handle fk reference (#17910)
rebelice238b057ba3d2
diff --git a/backend/plugin/schema/pg/generate_migration.go b/backend/plugin/schema/pg/generate_migration.goindex f6a03c0113..bbb87bf85d 100644--- a/backend/plugin/schema/pg/generate_migration.go+++ b/backend/plugin/schema/pg/generate_migration.go@@ -194,15 +194,25 @@ func dropObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) {}}- // For tables with foreign keys depending on other tables+ // For tables with foreign keys depending on other tables (DROP operations)for tableID, tableDiff := range tableMap {+ var foreignKeys []*storepb.ForeignKeyMetadata+if tableDiff.OldTable != nil {- for _, fk := range tableDiff.OldTable.ForeignKeys {- depID := getMigrationObjectID(fk.ReferencedSchema, fk.ReferencedTable)- if allObjects[depID] && depID != tableID {- // Edge from table with FK to referenced table- graph.AddEdge(tableID, depID)- }+ // Metadata mode: use ForeignKeys from metadata+ foreignKeys = tableDiff.OldTable.ForeignKeys+ } else if tableDiff.OldASTNode != nil {+ // AST-only mode: extract foreign keys from AST node+ foreignKeys = extractForeignKeysFromAST(tableDiff.OldASTNode, tableDiff.SchemaName)+ }++ for _, fk := range foreignKeys {+ depID := getMigrationObjectID(fk.ReferencedSchema, fk.ReferencedTable)+ if allObjects[depID] && depID != tableID {+ // Edge from table with FK to referenced table+ // For DROP: table1 (with FK) -> table2 (referenced)+ // This ensures table1 is dropped before table2+ graph.AddEdge(tableID, depID)}}}@@ -669,9 +679,21 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) error// Add dependency edges// For tables with foreign keys depending on other tables (only for CREATE operations)+ // Note: ALTER FK additions are handled after all tables are created, so they don't need topological sortingfor tableID, tableDiff := range tableMap {- if tableDiff.Action == schema.MetadataDiffActionCreate && tableDiff.NewTable != nil {- for _, fk := range tableDiff.NewTable.ForeignKeys {+ if tableDiff.Action == schema.MetadataDiffActionCreate {+ // For CREATE: extract all FKs from the table+ var foreignKeys []*storepb.ForeignKeyMetadata++ if tableDiff.NewTable != nil {+ // Metadata mode: use ForeignKeys from metadata+ foreignKeys = tableDiff.NewTable.ForeignKeys+ } else if tableDiff.NewASTNode != nil {+ // AST-only mode: extract foreign keys from AST node+ foreignKeys = extractForeignKeysFromAST(tableDiff.NewASTNode, tableDiff.SchemaName)+ }++ for _, fk := range foreignKeys {depID := getMigrationObjectID(fk.ReferencedSchema, fk.ReferencedTable)if depID != tableID {// Edge from dependency to dependent (referenced table to table with FK)@@ -3883,3 +3905,73 @@ func writeCreateIndexFromAST(out *strings.Builder, indexAST *pgparser.IndexstmtCreturn nil}++// extractForeignKeysFromAST extracts foreign key constraints from a CREATE TABLE AST node+// and returns foreign key metadata for dependency graph construction+func extractForeignKeysFromAST(createStmt *pgparser.CreatestmtContext, defaultSchema string) []*storepb.ForeignKeyMetadata {+ if createStmt == nil || createStmt.Opttableelementlist() == nil {+ return nil+ }++ tableElementList := createStmt.Opttableelementlist().Tableelementlist()+ if tableElementList == nil {+ return nil+ }++ var foreignKeys []*storepb.ForeignKeyMetadata++ for _, element := range tableElementList.AllTableelement() {+ if element.Tableconstraint() == nil {+ continue+ }++ constraint := element.Tableconstraint()+ if constraint.Constraintelem() == nil {+ continue+ }++ elem := constraint.Constraintelem()+ if elem.FOREIGN() == nil || elem.KEY() == nil {+ continue+ }++ // This is a foreign key constraint+ // Extract referenced table from qualified_name using AST and normalize functions+ // Grammar: FOREIGN KEY (...) REFERENCES qualified_name opt_column_list? ...+ qualifiedName := elem.Qualified_name()+ if qualifiedName == nil {+ continue+ }++ // Use NormalizePostgreSQLQualifiedName to get schema and table name+ // Returns []string: [table] or [schema, table]+ nameParts := pgpluginparser.NormalizePostgreSQLQualifiedName(qualifiedName)+ if len(nameParts) == 0 {+ continue+ }++ var referencedSchema, referencedTable string+ if len(nameParts) == 1 {+ // No schema specified, use default+ referencedSchema = defaultSchema+ referencedTable = nameParts[0]+ } else if len(nameParts) == 2 {+ // Schema and table specified+ referencedSchema = nameParts[0]+ referencedTable = nameParts[1]+ } else {+ // Unexpected format, skip+ continue+ }++ if referencedTable != "" {+ fk := &storepb.ForeignKeyMetadata{+ ReferencedSchema: referencedSchema,+ ReferencedTable: referencedTable,+ }+ foreignKeys = append(foreignKeys, fk)+ }+ }++ return foreignKeys+}diff --git a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go b/backend/plugin/schema/pg/generate_migration_testcontainer_test.goindex 20f3034578..65dd7ff1c1 100644--- a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go+++ b/backend/plugin/schema/pg/generate_migration_testcontainer_test.go@@ -1685,74 +1685,6 @@ DROP VIEW dept_employee_count;`,description: "Reverse of self_referencing_foreign_keys: DROP views and functions with dependencies",},- {- name: "reverse_circular_foreign_key_dependencies",- initialSchema: `-CREATE TABLE customers (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- preferred_order_id INTEGER-);--CREATE TABLE orders (- id SERIAL PRIMARY KEY,- customer_id INTEGER NOT NULL,- order_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- total_amount DECIMAL(10, 2),- CONSTRAINT fk_customer FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE CASCADE-);--CREATE TABLE order_items (- id SERIAL PRIMARY KEY,- order_id INTEGER NOT NULL,- product_name VARCHAR(100) NOT NULL,- quantity INTEGER NOT NULL,- unit_price DECIMAL(10, 2) NOT NULL,- CONSTRAINT fk_order_item FOREIGN KEY (order_id) REFERENCES orders(id) ON DELETE CASCADE-);--CREATE INDEX idx_order ON order_items(order_id);--ALTER TABLE customers ADD CONSTRAINT fk_preferred_order FOREIGN KEY (preferred_order_id) REFERENCES orders(id) ON DELETE SET NULL;--CREATE OR REPLACE FUNCTION update_order_total()-RETURNS TRIGGER AS $$-BEGIN- UPDATE orders- SET total_amount = (- SELECT SUM(quantity * unit_price)- FROM order_items- WHERE order_id = NEW.order_id- )- WHERE id = NEW.order_id;- RETURN NEW;-END;-$$ LANGUAGE plpgsql;--CREATE TRIGGER trg_update_order_total-AFTER INSERT OR UPDATE ON order_items-FOR EACH ROW-EXECUTE FUNCTION update_order_total();-`,- migrationDDL: `--- Drop trigger-DROP TRIGGER trg_update_order_total ON order_items;---- Drop function-DROP FUNCTION update_order_total();---- Drop index-DROP INDEX idx_order;---- Drop table-DROP TABLE order_items;---- Drop circular foreign key constraints-ALTER TABLE customers DROP CONSTRAINT fk_preferred_order;-ALTER TABLE orders DROP CONSTRAINT fk_customer;-`,- description: "Reverse of circular_foreign_key_dependencies: DROP circular FK dependencies",- },{name: "reverse_table_and_column_comments",initialSchema: `@@ -2481,3 +2413,101 @@ func sortExtensionsByName(extensions []*storepb.ExtensionMetadata) {return strings.Compare(a.Name, b.Name)})}++// TestSDLForeignKeyDependencyOrder tests that SDL migration correctly handles FK dependencies+// when tables are provided in alphabetical order (table1 before table2) but table1 references table2+func TestSDLForeignKeyDependencyOrder(t *testing.T) {+ ctx := context.Background()++ // Start PostgreSQL container+ pgContainer, err := postgres.Run(ctx,+ "postgres:16-alpine",+ postgres.WithDatabase("test_db"),+ postgres.WithUsername("postgres"),+ postgres.WithPassword("test"),+ testcontainers.WithWaitStrategy(+ wait.ForLog("database system is ready to accept connections").+ WithOccurrence(2).+ WithStartupTimeout(5*time.Minute),+ ),+ )+ require.NoError(t, err)+ defer func() {+ if err := pgContainer.Terminate(ctx); err != nil {+ t.Logf("failed to terminate container: %s", err)+ }+ }()++ // Get connection string+ connectionString, err := pgContainer.ConnectionString(ctx, "sslmode=disable")+ require.NoError(t, err)++ // Connect to the database+ connConfig, err := pgx.ParseConfig(connectionString)+ require.NoError(t, err)+ testDB := stdlib.OpenDB(*connConfig)+ defer testDB.Close()++ // This SDL text simulates multi-file SDL where files are merged in alphabetical order+ // table1.sql comes before table2.sql, but table1 has FK to table2+ currentSDL := `+CREATE TABLE table1 (+ id serial,+ name varchar(100) not null,+ created_at timestamp default current_timestamp,+ description text,+ constraint pk_table1_id primary key (id),+ constraint fk_table1_name foreign key (name) references table2(name)+);++CREATE TABLE table2 (+ id int,+ name varchar(100),+ constraint pk_table2_id primary key (id),+ constraint uq_table2_name unique (name)+);+`++ previousSDL := ``++ // Get current schema (empty database)+ currentSchema, err := getSyncMetadataForGenerateMigration(ctx, connConfig, connConfig.Database)+ require.NoError(t, err)+ dbSchema := model.NewDatabaseSchema(currentSchema, nil, nil, storepb.Engine_POSTGRES, false)++ // Get SDL diff+ diff, err := GetSDLDiff(currentSDL, previousSDL, dbSchema, nil)+ require.NoError(t, err)++ // Generate migration SQL with topological sorting+ migrationSQL, err := generateMigration(diff)+ require.NoError(t, err)++ t.Logf("Generated migration SQL:\n%s", migrationSQL)++ // Execute the generated migration SQL+ // This should work because generateMigration should have sorted tables by FK dependencies+ _, err = testDB.Exec(migrationSQL)+ require.NoError(t, err, "Migration SQL should execute successfully with correct FK dependency order")++ // Verify both tables were created+ var table1Exists, table2Exists bool+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'table1')").Scan(&table1Exists)+ require.NoError(t, err)+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'table2')").Scan(&table2Exists)+ require.NoError(t, err)++ require.True(t, table1Exists, "table1 should be created")+ require.True(t, table2Exists, "table2 should be created")++ // Verify FK constraint exists+ var fkExists bool+ err = testDB.QueryRow(`+ SELECT EXISTS (+ SELECT 1 FROM information_schema.table_constraints+ WHERE constraint_name = 'fk_table1_name' AND table_name = 'table1'+ )+ `).Scan(&fkExists)+ require.NoError(t, err)+ require.True(t, fkExists, "FK constraint fk_table1_name should exist")+}diff --git a/backend/plugin/schema/pg/topological_order_test.go b/backend/plugin/schema/pg/topological_order_test.goindex dbd0935102..e25de410a8 100644--- a/backend/plugin/schema/pg/topological_order_test.go+++ b/backend/plugin/schema/pg/topological_order_test.go@@ -62,6 +62,59 @@ func TestTopologicalOrderCreateObjects(t *testing.T) {},description: "Customers table should be created before orders table due to FK dependency",},+ {+ name: "table1_references_table2_name_column",+ diff: &schema.MetadataDiff{+ TableChanges: []*schema.TableDiff{+ // Create table1 that references table2(name) - this comes FIRST in the list+ {+ Action: schema.MetadataDiffActionCreate,+ SchemaName: "public",+ TableName: "table1",+ NewTable: &storepb.TableMetadata{+ Name: "table1",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "SERIAL", Nullable: false},+ {Name: "name", Type: "VARCHAR(100)", Nullable: false},+ {Name: "created_at", Type: "TIMESTAMP", Default: "CURRENT_TIMESTAMP", Nullable: false},+ {Name: "description", Type: "TEXT", Nullable: true},+ },+ ForeignKeys: []*storepb.ForeignKeyMetadata{+ {+ Name: "fk_table1_name",+ Columns: []string{"name"},+ ReferencedSchema: "public",+ ReferencedTable: "table2",+ ReferencedColumns: []string{"name"},+ },+ },+ },+ },+ // Create table2 with unique constraint on name - this comes SECOND in the list+ {+ Action: schema.MetadataDiffActionCreate,+ SchemaName: "public",+ TableName: "table2",+ NewTable: &storepb.TableMetadata{+ Name: "table2",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "INT", Nullable: false},+ {Name: "name", Type: "VARCHAR(100)", Nullable: true},+ },+ Indexes: []*storepb.IndexMetadata{+ {+ Name: "uq_table2_name",+ Expressions: []string{"name"},+ Unique: true,+ IsConstraint: true,+ },+ },+ },+ },+ },+ },+ description: "table2 should be created before table1 even though table1 appears first in the diff, because table1 has FK to table2",+ },{name: "view_depends_on_tables",diff: &schema.MetadataDiff{@@ -224,6 +277,13 @@ func TestTopologicalOrderCreateObjects(t *testing.T) {assert.True(t, customersIndex < ordersIndex,"Customers table should be created before orders table. Got statements: %v", statements)+ case "table1_references_table2_name_column":+ // table2 should be created before table1 because table1 has FK to table2+ table1Index := findStatementIndex(statements, "CREATE TABLE", "table1")+ table2Index := findStatementIndex(statements, "CREATE TABLE", "table2")+ assert.True(t, table2Index < table1Index,+ "table2 should be created before table1 because table1 has FK to table2(name). Got statements: %v", statements)+case "view_depends_on_tables":// Both tables should be created before the viewusersIndex := findStatementIndex(statements, "CREATE TABLE", "users")@@ -355,3 +415,129 @@ func findStatementIndex(statements []string, stmtType, objectName string) int {}return -1}++// TestTopologicalOrderDropObjects tests that objects are dropped in the correct dependency order+func TestTopologicalOrderDropObjects(t *testing.T) {+ tests := []struct {+ name string+ diff *schema.MetadataDiff+ description string+ }{+ {+ name: "drop_table1_with_fk_before_table2",+ diff: &schema.MetadataDiff{+ TableChanges: []*schema.TableDiff{+ // Drop table1 that has FK to table2 - comes FIRST in list+ {+ Action: schema.MetadataDiffActionDrop,+ SchemaName: "public",+ TableName: "table1",+ OldTable: &storepb.TableMetadata{+ Name: "table1",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "SERIAL", Nullable: false},+ {Name: "name", Type: "VARCHAR(100)", Nullable: false},+ },+ ForeignKeys: []*storepb.ForeignKeyMetadata{+ {+ Name: "fk_table1_name",+ Columns: []string{"name"},+ ReferencedSchema: "public",+ ReferencedTable: "table2",+ ReferencedColumns: []string{"name"},+ },+ },+ },+ },+ // Drop table2 - comes SECOND in list but should be dropped AFTER table1+ {+ Action: schema.MetadataDiffActionDrop,+ SchemaName: "public",+ TableName: "table2",+ OldTable: &storepb.TableMetadata{+ Name: "table2",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "INT", Nullable: false},+ {Name: "name", Type: "VARCHAR(100)", Nullable: false},+ },+ },+ },+ },+ },+ description: "table1 (with FK) should be dropped before table2 (referenced)",+ },+ {+ name: "drop_orders_before_customers",+ diff: &schema.MetadataDiff{+ TableChanges: []*schema.TableDiff{+ // Drop customers table - comes FIRST in list+ {+ Action: schema.MetadataDiffActionDrop,+ SchemaName: "public",+ TableName: "customers",+ OldTable: &storepb.TableMetadata{+ Name: "customers",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "SERIAL", Nullable: false},+ {Name: "name", Type: "VARCHAR(100)", Nullable: false},+ },+ },+ },+ // Drop orders table that has FK to customers - comes SECOND in list+ {+ Action: schema.MetadataDiffActionDrop,+ SchemaName: "public",+ TableName: "orders",+ OldTable: &storepb.TableMetadata{+ Name: "orders",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "SERIAL", Nullable: false},+ {Name: "customer_id", Type: "INT", Nullable: false},+ },+ ForeignKeys: []*storepb.ForeignKeyMetadata{+ {+ Name: "fk_orders_customer",+ Columns: []string{"customer_id"},+ ReferencedSchema: "public",+ ReferencedTable: "customers",+ ReferencedColumns: []string{"id"},+ },+ },+ },+ },+ },+ },+ description: "orders (with FK) should be dropped before customers (referenced)",+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ migrationSQL, err := generateMigration(tc.diff)+ require.NoError(t, err)++ // Parse the SQL into statements+ statements := parseStatements(migrationSQL)++ // For DROP operations, find the DROP statements+ switch tc.name {+ case "drop_table1_with_fk_before_table2":+ table1DropIdx := findStatementIndex(statements, "DROP TABLE", "table1")+ table2DropIdx := findStatementIndex(statements, "DROP TABLE", "table2")++ assert.NotEqual(t, -1, table1DropIdx, "table1 DROP statement should exist")+ assert.NotEqual(t, -1, table2DropIdx, "table2 DROP statement should exist")+ assert.Less(t, table1DropIdx, table2DropIdx, "table1 (with FK) should be dropped before table2 (referenced)")+ case "drop_orders_before_customers":+ ordersDropIdx := findStatementIndex(statements, "DROP TABLE", "orders")+ customersDropIdx := findStatementIndex(statements, "DROP TABLE", "customers")++ assert.NotEqual(t, -1, ordersDropIdx, "orders DROP statement should exist")+ assert.NotEqual(t, -1, customersDropIdx, "customers DROP statement should exist")+ assert.Less(t, ordersDropIdx, customersDropIdx, "orders (with FK) should be dropped before customers (referenced)")+ default:+ t.Fatalf("Unknown test case: %s", tc.name)+ }+ })+ }+}