Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: refactor to match Postgres pg_dump behavior (#17661)
Vincent Huang729564024e76
diff --git a/backend/plugin/schema/pg/get_database_definition.go b/backend/plugin/schema/pg/get_database_definition.goindex 8295515907..a2857cc750 100644--- a/backend/plugin/schema/pg/get_database_definition.go+++ b/backend/plugin/schema/pg/get_database_definition.go@@ -112,22 +112,36 @@ func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.Da}}- // Mapping from table ID to sequence metadata.- // Construct none owner column sequences first.- sequenceMap := make(map[string][]*storepb.SequenceMetadata)+ // Create non-identity sequences before tables to prevent errors when tables reference+ // sequences owned by other tables. Identity sequences are created via writeColumnIdentityGeneration.+ // Build identity column map once for O(m*c) instead of O(s*m*c) when checking each sequence.+ identityColumnMap := buildIdentityColumnMap(metadata)++ sequenceOwnershipMap := make(map[string][]*storepb.SequenceMetadata)for _, schema := range metadata.Schemas {for _, sequence := range schema.Sequences {if sequence.SkipDump {continue}- if sequence.OwnerTable == "" || sequence.OwnerColumn == "" {- if err := writeCreateSequence(&buf, schema.Name, sequence); err != nil {- return "", err- }+ // Check if this sequence belongs to an identity column using the prebuilt map+ isIdentity := false+ if sequence.OwnerTable != "" && sequence.OwnerColumn != "" {+ key := schema.Name + "." + sequence.OwnerTable + "." + sequence.OwnerColumn+ isIdentity = identityColumnMap[key]+ }++ if isIdentity {+ tableID := getObjectID(schema.Name, sequence.OwnerTable)+ sequenceOwnershipMap[tableID] = append(sequenceOwnershipMap[tableID], sequence)continue}- tableID := getObjectID(schema.Name, sequence.OwnerTable)- sequenceMap[tableID] = append(sequenceMap[tableID], sequence)+ if err := writeCreateSequence(&buf, schema.Name, sequence); err != nil {+ return "", err+ }+ if sequence.OwnerTable != "" && sequence.OwnerColumn != "" {+ tableID := getObjectID(schema.Name, sequence.OwnerTable)+ sequenceOwnershipMap[tableID] = append(sequenceOwnershipMap[tableID], sequence)+ }}}@@ -247,7 +261,7 @@ func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.Dacontinue}if table, ok := tableMap[objectID]; ok {- if err := writeTable(&buf, getSchemaNameFromID(objectID), table, sequenceMap[objectID]); err != nil {+ if err := writeTable(&buf, getSchemaNameFromID(objectID), table, sequenceOwnershipMap[objectID]); err != nil {return "", err}}@@ -389,21 +403,35 @@ func GetSchemaDefinition(schema *storepb.SchemaMetadata) (string, error) {}}- // Mapping from table ID to sequence metadata.- // Construct none owner column sequences first.- sequenceMap := make(map[string][]*storepb.SequenceMetadata)+ // Create non-identity sequences before tables to prevent errors when tables reference+ // sequences owned by other tables. Identity sequences are created via writeColumnIdentityGeneration.+ // Build identity column map once for O(m*c) instead of O(s*m*c) when checking each sequence.+ identityColumnMap := buildIdentityColumnMapForSchema(schema)++ sequenceOwnershipMap := make(map[string][]*storepb.SequenceMetadata)for _, sequence := range schema.Sequences {if sequence.SkipDump {continue}- if sequence.OwnerTable == "" || sequence.OwnerColumn == "" {- if err := writeCreateSequence(&buf, schema.Name, sequence); err != nil {- return "", err- }+ // Check if this sequence belongs to an identity column using the prebuilt map+ isIdentity := false+ if sequence.OwnerTable != "" && sequence.OwnerColumn != "" {+ key := sequence.OwnerTable + "." + sequence.OwnerColumn+ isIdentity = identityColumnMap[key]+ }++ if isIdentity {+ tableID := getObjectID(schema.Name, sequence.OwnerTable)+ sequenceOwnershipMap[tableID] = append(sequenceOwnershipMap[tableID], sequence)continue}- tableID := getObjectID(schema.Name, sequence.OwnerTable)- sequenceMap[tableID] = append(sequenceMap[tableID], sequence)+ if err := writeCreateSequence(&buf, schema.Name, sequence); err != nil {+ return "", err+ }+ if sequence.OwnerTable != "" && sequence.OwnerColumn != "" {+ tableID := getObjectID(schema.Name, sequence.OwnerTable)+ sequenceOwnershipMap[tableID] = append(sequenceOwnershipMap[tableID], sequence)+ }}// Construct tables, views and materialized views.@@ -514,7 +542,7 @@ func GetSchemaDefinition(schema *storepb.SchemaMetadata) (string, error) {continue}if table, ok := tableMap[objectID]; ok {- if err := writeTable(&buf, getSchemaNameFromID(objectID), table, sequenceMap[objectID]); err != nil {+ if err := writeTable(&buf, getSchemaNameFromID(objectID), table, sequenceOwnershipMap[objectID]); err != nil {return "", err}}@@ -1381,6 +1409,45 @@ func writeCreateTable(out io.Writer, schema string, tableName string, columns []return err}+// isIdentityColumn checks if a column is an identity column.+func isIdentityColumn(column *storepb.ColumnMetadata) bool {+ return column.IdentityGeneration == storepb.ColumnMetadata_ALWAYS ||+ column.IdentityGeneration == storepb.ColumnMetadata_BY_DEFAULT+}++// buildIdentityColumnMap builds a map of identity columns for the entire database.+// Key format: "schemaName.tableName.columnName" -> true+// This is O(m*c) which is done once, vs O(s*m*c) when checking each sequence individually.+func buildIdentityColumnMap(metadata *storepb.DatabaseSchemaMetadata) map[string]bool {+ identityMap := make(map[string]bool)+ for _, schema := range metadata.Schemas {+ for _, table := range schema.Tables {+ for _, column := range table.Columns {+ if isIdentityColumn(column) {+ key := schema.Name + "." + table.Name + "." + column.Name+ identityMap[key] = true+ }+ }+ }+ }+ return identityMap+}++// buildIdentityColumnMapForSchema builds a map of identity columns for a single schema.+// Key format: "tableName.columnName" -> true+func buildIdentityColumnMapForSchema(schema *storepb.SchemaMetadata) map[string]bool {+ identityMap := make(map[string]bool)+ for _, table := range schema.Tables {+ for _, column := range table.Columns {+ if isIdentityColumn(column) {+ key := table.Name + "." + column.Name+ identityMap[key] = true+ }+ }+ }+ return identityMap+}+func splitSequencesByIdentityOrNot(table *storepb.TableMetadata, sequences []*storepb.SequenceMetadata) ([]storepb.ColumnMetadata_IdentityGeneration, []*storepb.SequenceMetadata, []*storepb.SequenceMetadata) {columnMap := make(map[string]*storepb.ColumnMetadata)for _, column := range table.Columns {@@ -1391,7 +1458,7 @@ func splitSequencesByIdentityOrNot(table *storepb.TableMetadata, sequences []*stvar nonIdentitySequences []*storepb.SequenceMetadatafor _, sequence := range sequences {if column, ok := columnMap[sequence.OwnerColumn]; ok {- if column.IdentityGeneration == storepb.ColumnMetadata_ALWAYS || column.IdentityGeneration == storepb.ColumnMetadata_BY_DEFAULT {+ if isIdentityColumn(column) {generationType = append(generationType, column.IdentityGeneration)identitySequences = append(identitySequences, sequence)continue@@ -1404,11 +1471,6 @@ func splitSequencesByIdentityOrNot(table *storepb.TableMetadata, sequences []*stfunc writeTable(out io.Writer, schema string, table *storepb.TableMetadata, sequences []*storepb.SequenceMetadata) error {generationTypes, identitySequences, nonIdentitySequences := splitSequencesByIdentityOrNot(table, sequences)- for _, sequence := range nonIdentitySequences {- if err := writeCreateSequence(out, schema, sequence); err != nil {- return err- }- }if err := writeCreateTable(out, schema, table.Name, table.Columns, table.CheckConstraints); err != nil {return err@@ -2123,31 +2185,29 @@ func getSDLFormat(metadata *storepb.DatabaseSchemaMetadata) (string, error) {}}- // Mapping from table ID to sequence metadata for table-owned sequences- sequenceMap := make(map[string][]*storepb.SequenceMetadata)+ // Write all sequences before tables to ensure they exist before any table references them.+ sequenceOwnershipMap := make(map[string][]*storepb.SequenceMetadata)for _, schema := range metadata.Schemas {if schema.SkipDump {continue}- // Write independent sequences first (not owned by any table)for _, sequence := range schema.Sequences {if sequence.SkipDump {continue}- if sequence.OwnerTable == "" || sequence.OwnerColumn == "" {- if err := writeSequenceSDL(&buf, schema.Name, sequence); err != nil {- return "", err- }- if _, err := buf.WriteString(";\n\n"); err != nil {- return "", err- }- continue+ if err := writeSequenceSDL(&buf, schema.Name, sequence); err != nil {+ return "", err+ }+ if _, err := buf.WriteString(";\n\n"); err != nil {+ return "", err+ }+ // Track sequences with owners for later ownership establishment+ if sequence.OwnerTable != "" && sequence.OwnerColumn != "" {+ tableID := getObjectID(schema.Name, sequence.OwnerTable)+ sequenceOwnershipMap[tableID] = append(sequenceOwnershipMap[tableID], sequence)}- // Table-owned sequences will be handled with their tables- tableID := getObjectID(schema.Name, sequence.OwnerTable)- sequenceMap[tableID] = append(sequenceMap[tableID], sequence)}}@@ -2156,30 +2216,12 @@ func getSDLFormat(metadata *storepb.DatabaseSchemaMetadata) (string, error) {continue}- // Write tables with their owned sequences+ // Write tables (sequences are already written above)for _, table := range schema.Tables {if table.SkipDump {continue}- tableID := getObjectID(schema.Name, table.Name)- tableSequences := sequenceMap[tableID]-- // Write table-owned sequences that are not identity sequences- if len(tableSequences) > 0 {- _, identitySequences, nonIdentitySequences := splitSequencesByIdentityOrNot(table, tableSequences)- for _, sequence := range nonIdentitySequences {- if err := writeSequenceSDL(&buf, schema.Name, sequence); err != nil {- return "", err- }- if _, err := buf.WriteString(";\n\n"); err != nil {- return "", err- }- }- // Identity sequences are handled as part of column definition- _ = identitySequences- }-// Write CREATE TABLE statementif err := writeCreateTableSDL(&buf, schema.Name, table); err != nil {return "", errdiff --git a/backend/plugin/schema/pg/get_database_definition_sdl_test.go b/backend/plugin/schema/pg/get_database_definition_sdl_test.goindex ae52c06168..b7c40bdd5f 100644--- a/backend/plugin/schema/pg/get_database_definition_sdl_test.go+++ b/backend/plugin/schema/pg/get_database_definition_sdl_test.go@@ -637,10 +637,10 @@ $$;},expected: `CREATE SEQUENCE "public"."independent_seq";-CREATE SEQUENCE "public"."order_seq" AS integer START WITH 1000 INCREMENT BY 10 MINVALUE 1000 MAXVALUE 999999 CYCLE;-CREATE SEQUENCE "public"."user_id_seq";+CREATE SEQUENCE "public"."order_seq" AS integer START WITH 1000 INCREMENT BY 10 MINVALUE 1000 MAXVALUE 999999 CYCLE;+CREATE TABLE "public"."users" ("id" INTEGER DEFAULT nextval('user_id_seq'::regclass) NOT NULL,"name" VARCHAR(255) NOT NULLdiff --git a/backend/plugin/schema/pg/get_database_definition_testcontainer_test.go b/backend/plugin/schema/pg/get_database_definition_testcontainer_test.goindex 89a8d014ce..16314a1ddf 100644--- a/backend/plugin/schema/pg/get_database_definition_testcontainer_test.go+++ b/backend/plugin/schema/pg/get_database_definition_testcontainer_test.go@@ -74,6 +74,37 @@ CREATE TABLE items (status status_enum DEFAULT 'pending',user_mood mood);+`,+ },+ {+ name: "owned_sequence_dependency_order",+ ddl: `+-- This reproduces the customer's reported issue+-- Create a sequence that will be owned by a column+CREATE SEQUENCE history_cost_changes_id_seq+ AS bigint+ START WITH 1+ INCREMENT BY 1+ MINVALUE 1+ MAXVALUE 9223372036854775807+ NO CYCLE;++-- Create table using the sequence in DEFAULT clause+CREATE TABLE history_cost_changes (+ id bigint DEFAULT nextval('history_cost_changes_id_seq'::regclass) NOT NULL PRIMARY KEY,+ amount numeric(10,2),+ description text,+ created_at timestamp DEFAULT CURRENT_TIMESTAMP+);++-- Make the sequence owned by the column (critical for reproducing the bug!)+ALTER SEQUENCE history_cost_changes_id_seq OWNED BY history_cost_changes.id;++-- Create another table that also references the same sequence (edge case)+CREATE TABLE history_audit (+ audit_id bigint DEFAULT nextval('history_cost_changes_id_seq'::regclass) NOT NULL,+ change_type varchar(50)+);`,},{