Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(pg): delete GetDatabaseMetadata and unify migration to DiffSDLMigration [BYT-9010] (#19744)
rebelice8049b14fb776
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.goindex fc07ed083c..917ec191d1 100644--- a/backend/api/v1/database_service.go+++ b/backend/api/v1/database_service.go@@ -706,30 +706,68 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connect// DiffSchema diff the database schema.func (s *DatabaseService) DiffSchema(ctx context.Context, req *connect.Request[v1pb.DiffSchemaRequest]) (*connect.Response[v1pb.DiffSchemaResponse], error) {- // Use unified SDL-based approach for all scenarios+ engine, err := s.getParserEngine(ctx, req.Msg)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get parser engine"))+ }++ // PG/CockroachDB: use omni SDL diff path.+ if engine == storepb.Engine_POSTGRES || engine == storepb.Engine_COCKROACHDB {+ migrationSQL, err := s.diffSchemaViaSDL(ctx, req.Msg, engine)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))+ }+ return connect.NewResponse(&v1pb.DiffSchemaResponse{Diff: migrationSQL}), nil+ }++ // Other engines: legacy metadata diff path.sourceDBSchema, err := s.getSourceDBMetadata(ctx, req.Msg)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get source schema"))}-targetDBSchema, err := s.getTargetDBMetadata(ctx, req.Msg)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get target schema"))}-- engine, err := s.getParserEngine(ctx, req.Msg)+ migrationSQL, err := schema.DiffMigration(engine, sourceDBSchema, targetDBSchema)if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get parser engine"))+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))}+ return connect.NewResponse(&v1pb.DiffSchemaResponse{Diff: migrationSQL}), nil+}- migrationSQL, err := schema.DiffMigration(engine, sourceDBSchema, targetDBSchema)+// diffSchemaViaSDL handles PG/CockroachDB DiffSchema using omni SDL diff.+func (s *DatabaseService) diffSchemaViaSDL(ctx context.Context, req *v1pb.DiffSchemaRequest, engine storepb.Engine) (string, error) {+ sourceMetadata, err := s.getSourceDBMetadata(ctx, req)if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))+ return "", err+ }+ sourceSDL, err := schema.MetadataToSDL(engine, sourceMetadata)+ if err != nil {+ return "", errors.Wrap(err, "failed to generate source SDL")}- return connect.NewResponse(&v1pb.DiffSchemaResponse{- Diff: migrationSQL,- }), nil+ // Resolve target schema text.+ var targetSDL string+ switch {+ case req.GetSchema() != "":+ // Schema text from GetDatabaseSchema (raw dump) — passed directly.+ // pgDiffSDLMigration handles LoadSDL/LoadSQL fallback internally.+ targetSDL = req.GetSchema()+ case req.GetChangelog() != "":+ targetMetadata, err := s.getSourceDBMetadata(ctx, &v1pb.DiffSchemaRequest{Name: req.GetChangelog()})+ if err != nil {+ return "", errors.Wrap(err, "failed to resolve target changelog")+ }+ targetSDL, err = schema.MetadataToSDL(engine, targetMetadata)+ if err != nil {+ return "", errors.Wrap(err, "failed to generate target SDL")+ }+ default:+ return "", errors.Errorf("target must be either schema text or changelog")+ }++ return schema.DiffSDLMigration(engine, sourceSDL, targetSDL)}func (s *DatabaseService) getSourceDBMetadata(ctx context.Context, request *v1pb.DiffSchemaRequest) (*model.DatabaseMetadata, error) {diff --git a/backend/plugin/schema/pg/camelcase_index_test.go b/backend/plugin/schema/pg/camelcase_index_test.goindex 6e9e9c51b3..a96f70717a 100644--- a/backend/plugin/schema/pg/camelcase_index_test.go+++ b/backend/plugin/schema/pg/camelcase_index_test.go@@ -33,73 +33,6 @@ func TestQuoteIndexExpression(t *testing.T) {}// TestCamelCaseIndexColumnQuoting_SyncSchema reproduces https://github.com/bytebase/bytebase/issues/19348-// via the Sync Schema path: the source schema is exported (with proper quoting), then-// parsed into metadata by GetDatabaseMetadata (which calls generateIndexDefinition).-// The parsed metadata's Definition must preserve quoting for CamelCase columns.-func TestCamelCaseIndexColumnQuoting_SyncSchema(t *testing.T) {- // This is the exported schema from the source database (pg_get_indexdef quotes correctly).- sourceSchemaSQL := `-CREATE TABLE "public"."ba_account" (- "userId" text NOT NULL,- "accountName" text-);--CREATE INDEX "ba_account_userid_idx" ON "public"."ba_account" ("userId");-`-- // Parse the source schema into metadata (this is what getTargetDBMetadata does).- parsedMetadata, err := GetDatabaseMetadata(sourceSchemaSQL)- require.NoError(t, err)-- // Verify the parsed index Definition has quoted "userId".- var indexDef string- for _, s := range parsedMetadata.Schemas {- for _, tbl := range s.Tables {- for _, idx := range tbl.Indexes {- if idx.Name == "ba_account_userid_idx" {- indexDef = idx.Definition- }- }- }- }- require.NotEmpty(t, indexDef, "index definition should be set")- t.Logf("Parsed index Definition: %s", indexDef)- require.Contains(t, indexDef, `"userId"`, "CamelCase column must be quoted in parsed Definition")-- // Now simulate the full Sync Schema diff: target has no index, source has it.- oldMetadata := &storepb.DatabaseSchemaMetadata{- Name: "testdb",- Schemas: []*storepb.SchemaMetadata{- {- Name: "public",- Tables: []*storepb.TableMetadata{- {- Name: "ba_account",- Columns: []*storepb.ColumnMetadata{- {Name: "userId", Type: "text"},- {Name: "accountName", Type: "text"},- },- },- },- },- },- }-- oldSchema := model.NewDatabaseMetadata(oldMetadata, nil, nil, storepb.Engine_POSTGRES, false)- newSchema := model.NewDatabaseMetadata(parsedMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, oldSchema, newSchema)- require.NoError(t, err)- require.NotNil(t, diff)-- ddl, err := generateMigration(diff)- require.NoError(t, err)-- t.Logf("Generated DDL:\n%s", ddl)-- require.Contains(t, ddl, `"userId"`, "CamelCase column in CREATE INDEX must be quoted")-}-// TestCamelCaseConstraintQuoting_PrimaryKey verifies that PRIMARY KEY constraints// on CamelCase columns are properly quoted in generated ALTER TABLE statements.func TestCamelCaseConstraintQuoting_PrimaryKey(t *testing.T) {diff --git a/backend/plugin/schema/pg/extension_test.go b/backend/plugin/schema/pg/extension_test.goindex 8b618edb2a..c3a007438e 100644--- a/backend/plugin/schema/pg/extension_test.go+++ b/backend/plugin/schema/pg/extension_test.go@@ -319,118 +319,3 @@ func TestExtensionNotFilteredByArchiveSchemaFilter(t *testing.T) {// 1. Incorrectly generates DROP EXTENSION even though the extension is identical in source and target// 2. Generates CREATE SCHEMA for a phantom schema named after the extension//-// Root causes:-// - Bug 1: EnterCreateextensionstmt doesn't parse VERSION from SQL → compareExtensions sees false mismatch-// - Bug 2: EnterCommentstmt treats COMMENT ON EXTENSION as COMMENT ON SCHEMA → creates phantom schema-func TestExtensionSchemaSyncBug(t *testing.T) {- t.Run("Bug1_version_not_parsed_from_SQL", func(t *testing.T) {- // The raw schema dump (produced by writeExtension) includes VERSION.- // GetDatabaseMetadata must parse it back correctly.- sql := `CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';`-- metadata, err := GetDatabaseMetadata(sql)- require.NoError(t, err)- require.NotNil(t, metadata)- require.Len(t, metadata.Extensions, 1, "Should have exactly 1 extension")-- ext := metadata.Extensions[0]- require.Equal(t, "pg_trgm", ext.Name)- require.Equal(t, "tecbatch", ext.Schema)- require.Equal(t, "1.6", ext.Version, "BUG: VERSION not parsed from CREATE EXTENSION statement")- })-- t.Run("Bug2_comment_on_extension_creates_phantom_schema", func(t *testing.T) {- // COMMENT ON EXTENSION should NOT create a schema entry.- sql := `CREATE SCHEMA IF NOT EXISTS "tecbatch";-CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';-COMMENT ON EXTENSION "pg_trgm" IS 'text similarity measurement and index searching based on trigrams';`-- metadata, err := GetDatabaseMetadata(sql)- require.NoError(t, err)- require.NotNil(t, metadata)-- // Should only have "public" (always created) and "tecbatch" schemas.- // Must NOT have a "pg_trgm" schema.- for _, s := range metadata.Schemas {- require.NotEqual(t, "pg_trgm", s.Name,- "BUG: COMMENT ON EXTENSION created a phantom schema named after the extension")- }-- // Extension description should be populated (not lost).- require.Len(t, metadata.Extensions, 1)- require.Equal(t, "text similarity measurement and index searching based on trigrams",- metadata.Extensions[0].Description,- "BUG: COMMENT ON EXTENSION description not stored in extension metadata")- })-- t.Run("Full_pipeline_no_false_diff", func(t *testing.T) {- // Simulate the exact Schema Sync flow:- // "old" = target DB metadata from live DB sync (has version + description)- // "new" = source DB metadata parsed from raw schema dump-- // --- "old" metadata: target DB (from getExtensions live query) ---- oldMetadata := &storepb.DatabaseSchemaMetadata{- Extensions: []*storepb.ExtensionMetadata{- {- Name: "pg_trgm",- Schema: "tecbatch",- Version: "1.6",- Description: "text similarity measurement and index searching based on trigrams",- },- },- Schemas: []*storepb.SchemaMetadata{- {Name: "public"},- {Name: "tecbatch"},- },- }-- // --- "new" SQL: source DB schema dump (from writeExtension) ---- sourceSQL := `CREATE SCHEMA IF NOT EXISTS "tecbatch";--CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';--COMMENT ON EXTENSION "pg_trgm" IS 'text similarity measurement and index searching based on trigrams';--CREATE OR REPLACE FUNCTION "tecbatch"."contains_one_of"(str character varying, match_strs character varying[], match_case boolean)- RETURNS boolean- LANGUAGE plpgsql- IMMUTABLE-AS $function$-DECLARE- result boolean := false;-BEGIN- RETURN result;-END;-$function$;`-- // Parse source SQL (this is the buggy path)- newMetadata, err := GetDatabaseMetadata(sourceSQL)- require.NoError(t, err)-- // Build model objects for differ- oldModel := model.NewDatabaseMetadata(oldMetadata, nil, nil, storepb.Engine_POSTGRES, false)- newModel := model.NewDatabaseMetadata(newMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- // Run differ- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, oldModel, newModel)- require.NoError(t, err)-- // Generate migration DDL- migration, err := generateMigration(diff)- require.NoError(t, err)-- // The source and target have identical extensions → migration should be empty- // (or at most contain the function CREATE which is expected).- require.NotContains(t, migration, "DROP EXTENSION",- "BUG: Schema Sync incorrectly drops pg_trgm extension that exists in both databases")- require.NotContains(t, migration, `CREATE SCHEMA IF NOT EXISTS "pg_trgm"`,- "BUG: Schema Sync creates phantom pg_trgm schema from COMMENT ON EXTENSION")- require.NotContains(t, migration, "CREATE EXTENSION",- "BUG: Schema Sync recreates pg_trgm extension even though it's identical")-- // Log the migration for debugging if any assertions above fail- if t.Failed() {- t.Logf("Generated migration (should be empty or function-only):\n%s", migration)- }- })-}diff --git a/backend/plugin/schema/pg/get_database_definition.go b/backend/plugin/schema/pg/get_database_definition.goindex dee00c5cd0..f2873f7898 100644--- a/backend/plugin/schema/pg/get_database_definition.go+++ b/backend/plugin/schema/pg/get_database_definition.go@@ -6,15 +6,14 @@ import ("slices""strings"- "github.com/antlr4-go/antlr/v4""github.com/pkg/errors"- parser "github.com/bytebase/parser/postgresql"+ omnipg "github.com/bytebase/omni/pg"+ "github.com/bytebase/omni/pg/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg""github.com/bytebase/bytebase/backend/plugin/schema")@@ -37,14 +36,16 @@ SET row_security = off;)func init() {- schema.RegisterGetDatabaseDefinition(storepb.Engine_POSTGRES, GetDatabaseDefinition)- schema.RegisterGetSchemaDefinition(storepb.Engine_POSTGRES, GetSchemaDefinition)- schema.RegisterGetTableDefinition(storepb.Engine_POSTGRES, GetTableDefinition)- schema.RegisterGetViewDefinition(storepb.Engine_POSTGRES, GetViewDefinition)- schema.RegisterGetMaterializedViewDefinition(storepb.Engine_POSTGRES, GetMaterializedViewDefinition)- schema.RegisterGetFunctionDefinition(storepb.Engine_POSTGRES, GetFunctionDefinition)- schema.RegisterGetSequenceDefinition(storepb.Engine_POSTGRES, GetSequenceDefinition)- schema.RegisterGetMultiFileDatabaseDefinition(storepb.Engine_POSTGRES, GetMultiFileDatabaseDefinition)+ for _, engine := range []storepb.Engine{storepb.Engine_POSTGRES, storepb.Engine_COCKROACHDB} {+ schema.RegisterGetDatabaseDefinition(engine, GetDatabaseDefinition)+ schema.RegisterGetSchemaDefinition(engine, GetSchemaDefinition)+ schema.RegisterGetTableDefinition(engine, GetTableDefinition)+ schema.RegisterGetViewDefinition(engine, GetViewDefinition)+ schema.RegisterGetMaterializedViewDefinition(engine, GetMaterializedViewDefinition)+ schema.RegisterGetFunctionDefinition(engine, GetFunctionDefinition)+ schema.RegisterGetSequenceDefinition(engine, GetSequenceDefinition)+ schema.RegisterGetMultiFileDatabaseDefinition(engine, GetMultiFileDatabaseDefinition)+ }}func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (string, error) {@@ -2240,60 +2241,26 @@ func writeFunction(out io.Writer, schema string, function *storepb.FunctionMetadreturn nil}-// functionExtractor walks a PostgreSQL parse tree and captures the first-// CREATE FUNCTION/PROCEDURE statement it encounters.-type functionExtractor struct {- parser.BasePostgreSQLParserListener- result **parser.CreatefunctionstmtContext-}--func (e *functionExtractor) EnterCreatefunctionstmt(ctx *parser.CreatefunctionstmtContext) {- if e.result != nil && *e.result == nil {- *e.result = ctx- }-}--// isDefinitionProcedure checks if the definition string represents a PROCEDURE (not a FUNCTION)-// Returns true if it's a PROCEDURE, false if it's a FUNCTION-// This function uses AST-based parsing for robust detection+// isDefinitionProcedure checks if the definition string represents a PROCEDURE (not a FUNCTION).+// Uses omni parser: procedures have a DefElem{Defname: "isProcedure"} in Options.func isDefinitionProcedure(definition string) bool {if definition == "" {return false}-- // Parse the definition to get AST- parseResults, err := pgparser.ParsePostgreSQL(definition)- if err != nil {- // If parsing fails, fall back to string-based detection- // This should rarely happen for valid definitions- upperDef := strings.ToUpper(definition)- return strings.Contains(upperDef, " PROCEDURE ") ||- strings.HasPrefix(upperDef, "CREATE PROCEDURE") ||- strings.HasPrefix(upperDef, "CREATE OR REPLACE PROCEDURE")- }-- // For function/procedure definition, we expect exactly one statement- if len(parseResults) != 1 {+ stmts, err := omnipg.Parse(definition)+ if err != nil || len(stmts) == 0 {return false}-- tree := parseResults[0].Tree- if tree == nil {+ fn, ok := stmts[0].AST.(*ast.CreateFunctionStmt)+ if !ok || fn == nil || fn.Options == nil {return false}-- // Walk the AST to find CREATE FUNCTION/PROCEDURE statement- var result *parser.CreatefunctionstmtContext- extractor := &functionExtractor{result: &result}- antlr.NewParseTreeWalker().Walk(extractor, tree)-- if result == nil {- return false+ for _, item := range fn.Options.Items {+ if def, ok := item.(*ast.DefElem); ok && def.Defname == "isProcedure" {+ return true+ }}-- // Check if it's a PROCEDURE by examining the AST node- // CreatefunctionstmtContext has both FUNCTION() and PROCEDURE() methods- return result.PROCEDURE() != nil+ return false}func writeFunctionComment(out io.Writer, schema string, function *storepb.FunctionMetadata) error {@@ -3246,15 +3213,14 @@ func writeIndexInternal(out io.Writer, schema string, table string, index *store}// Write INCLUDE and WHERE clauses from the full index definition (from pg_get_indexdef).- // SQL syntax order: (keys) INCLUDE (cols) WHERE (predicate)if index.Definition != "" {- comparer := &PostgreSQLIndexComparer{}- if includeClause := comparer.ExtractIncludeClauseFromIndexDef(index.Definition); includeClause != "" {+ includeClause, whereClause := extractIndexClauses(index.Definition)+ if includeClause != "" {if _, err := fmt.Fprintf(out, " %s", includeClause); err != nil {return err}}- if whereClause := comparer.ExtractWhereClauseFromIndexDef(index.Definition); whereClause != "" {+ if whereClause != "" {if _, err := fmt.Fprintf(out, " WHERE %s", whereClause); err != nil {return err}@@ -4268,3 +4234,66 @@ func writeTriggerCommentSDL(out io.Writer, schemaName, tableName string, trigger_, err := io.WriteString(out, "\n\n")return err}++// extractIndexClauses parses a CREATE INDEX statement using the omni parser and+// extracts the INCLUDE clause and WHERE clause as strings.+func extractIndexClauses(definition string) (includeClause, whereClause string) {+ stmts, err := omnipg.Parse(definition)+ if err != nil || len(stmts) == 0 {+ return "", ""+ }+ idx, ok := stmts[0].AST.(*ast.IndexStmt)+ if !ok || idx == nil {+ return "", ""+ }++ // Extract INCLUDE columns using Loc to preserve original quoting.+ if idx.IndexIncludingParams != nil {+ var cols []string+ for _, item := range idx.IndexIncludingParams.Items {+ if elem, ok := item.(*ast.IndexElem); ok && elem.Loc.Start >= 0 && elem.Loc.End > elem.Loc.Start {+ cols = append(cols, definition[elem.Loc.Start:elem.Loc.End])+ }+ }+ if len(cols) > 0 {+ includeClause = "INCLUDE (" + strings.Join(cols, ", ") + ")"+ }+ }++ // Extract WHERE clause using AST node Loc for precise extraction.+ if idx.WhereClause != nil {+ loc := ast.NodeLoc(idx.WhereClause)+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(definition) {+ raw := definition[loc.Start:loc.End]+ // Strip outer parentheses if the entire predicate is wrapped.+ if strings.HasPrefix(raw, "(") && strings.HasSuffix(raw, ")") {+ inner := raw[1 : len(raw)-1]+ // Only strip if parentheses are balanced.+ if isBalancedParens(inner) {+ raw = inner+ }+ }+ whereClause = strings.TrimSpace(raw)+ }+ }++ return includeClause, whereClause+}++// isBalancedParens checks if a string has balanced parentheses.+func isBalancedParens(s string) bool {+ depth := 0+ for _, c := range s {+ switch c {+ case '(':+ depth+++ case ')':+ depth--+ if depth < 0 {+ return false+ }+ default:+ }+ }+ return depth == 0+}diff --git a/backend/plugin/schema/pg/get_database_metadata.go b/backend/plugin/schema/pg/get_database_metadata.godeleted file mode 100644index 2faefc77c5..0000000000--- a/backend/plugin/schema/pg/get_database_metadata.go+++ /dev/null@@ -1,2923 +0,0 @@-package pg--import (- "context"- "fmt"- "slices"- "strings"-- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/postgresql"- "github.com/pkg/errors"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/parser/base"- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"- "github.com/bytebase/bytebase/backend/plugin/schema"- "github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"- "github.com/bytebase/bytebase/backend/store/model"-)--func init() {- schema.RegisterGetDatabaseMetadata(storepb.Engine_POSTGRES, GetDatabaseMetadata)- schema.RegisterGetDatabaseMetadata(storepb.Engine_COCKROACHDB, GetDatabaseMetadata)-}--// GetDatabaseMetadata parses the SQL schema text and returns the database metadata.-func GetDatabaseMetadata(schemaText string) (*storepb.DatabaseSchemaMetadata, error) {- parseResults, err := pgparser.ParsePostgreSQL(schemaText)- if err != nil {- return nil, errors.Wrapf(err, "failed to parse PostgreSQL schema")- }-- extractor := &metadataExtractor{- currentDatabase: "",- currentSchema: "public",- schemas: make(map[string]*storepb.SchemaMetadata),- tables: make(map[tableKey]*storepb.TableMetadata),- partitionTables: make(map[tableKey]bool),- partitionExpressions: make(map[tableKey]string),- partitionTypes: make(map[tableKey]storepb.TablePartitionMetadata_Type),- sequences: make(map[string]*storepb.SequenceMetadata),- extensions: make(map[string]*storepb.ExtensionMetadata),- expressionComparer: ast.NewPostgreSQLExpressionComparer(),- triggers: make(map[string][]*storepb.TriggerMetadata),- }-- // Always ensure public schema exists- extractor.getOrCreateSchema("public")-- // Walk all parsed statements- for _, parseResult := range parseResults {- if parseResult.Tree != nil {- antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)- }- }-- if extractor.err != nil {- return nil, extractor.err- }-- // Build the final metadata structure- schemaMetadata := &storepb.DatabaseSchemaMetadata{- Name: extractor.currentDatabase,- }-- // Sort schemas for consistent output- var schemaNames []string- for name := range extractor.schemas {- schemaNames = append(schemaNames, name)- }- slices.Sort(schemaNames)-- for _, schemaName := range schemaNames {- schema := extractor.schemas[schemaName]-- // Filter out any tables that are actually materialized views- // This handles the case where materialized views are incorrectly classified as tables- filteredTables := []*storepb.TableMetadata{}- for _, table := range schema.Tables {- // Check if this table name exists as a materialized view- isMaterializedView := false- for _, mv := range schema.MaterializedViews {- if mv.Name == table.Name {- isMaterializedView = true- break- }- }-- // Only include in tables if it's not a materialized view- if !isMaterializedView {- filteredTables = append(filteredTables, table)- }- }- schema.Tables = filteredTables-- // Assign triggers to their respective tables- for _, table := range filteredTables {- tableKey := fmt.Sprintf("%s.%s", schemaName, table.Name)- if triggers, exists := extractor.triggers[tableKey]; exists {- table.Triggers = triggers- }- }-- // Also assign triggers to materialized views if any- for _, mv := range schema.MaterializedViews {- tableKey := fmt.Sprintf("%s.%s", schemaName, mv.Name)- if triggers, exists := extractor.triggers[tableKey]; exists {- mv.Triggers = triggers- }- }-- // Add sequences to the schema (including SERIAL-generated sequences)- var sequencesForSchema []*storepb.SequenceMetadata- for sequenceKey, sequence := range extractor.sequences {- // Parse sequence key format: "schemaname.sequencename"- parts := strings.SplitN(sequenceKey, ".", 2)- if len(parts) == 2 && parts[0] == schemaName {- sequencesForSchema = append(sequencesForSchema, sequence)- }- }- // Sort sequences for consistent output- slices.SortFunc(sequencesForSchema, func(a, b *storepb.SequenceMetadata) int {- return strings.Compare(a.Name, b.Name)- })- schema.Sequences = sequencesForSchema-- schemaMetadata.Schemas = append(schemaMetadata.Schemas, schema)- }-- // Add extensions- var extensionNames []string- for name := range extractor.extensions {- extensionNames = append(extensionNames, name)- }- slices.Sort(extensionNames)-- for _, name := range extensionNames {- schemaMetadata.Extensions = append(schemaMetadata.Extensions, extractor.extensions[name])- }-- // Extract view dependencies after all metadata is collected- extractViewDependencies(schemaMetadata)-- return schemaMetadata, nil-}--type tableKey struct {- schema string- table string-}--// metadataExtractor walks the parse tree and extracts metadata-type metadataExtractor struct {- *parser.BasePostgreSQLParserListener-- currentDatabase string- currentSchema string- schemas map[string]*storepb.SchemaMetadata- tables map[tableKey]*storepb.TableMetadata- partitionTables map[tableKey]bool- partitionExpressions map[tableKey]string- partitionTypes map[tableKey]storepb.TablePartitionMetadata_Type- sequences map[string]*storepb.SequenceMetadata- extensions map[string]*storepb.ExtensionMetadata- expressionComparer ast.ExpressionComparer- triggers map[string][]*storepb.TriggerMetadata // Map from table key to triggers- err error-}--// Helper function to get or create schema-func (e *metadataExtractor) getOrCreateSchema(schemaName string) *storepb.SchemaMetadata {- if schemaName == "" {- schemaName = "public"- }-- if schema, exists := e.schemas[schemaName]; exists {- return schema- }-- schema := &storepb.SchemaMetadata{- Name: schemaName,- Tables: []*storepb.TableMetadata{},- Views: nil,- MaterializedViews: nil,- Procedures: nil,- Functions: nil,- Sequences: nil,- EnumTypes: nil,- }- e.schemas[schemaName] = schema- return schema-}--// Helper function to get or create table-func (e *metadataExtractor) getOrCreateTable(schemaName, tableName string) *storepb.TableMetadata {- key := tableKey{- schema: schemaName,- table: tableName,- }-- if table, exists := e.tables[key]; exists {- return table- }-- table := &storepb.TableMetadata{- Name: tableName,- Columns: []*storepb.ColumnMetadata{},- Indexes: []*storepb.IndexMetadata{},- ForeignKeys: nil,- CheckConstraints: nil,- Triggers: nil,- Partitions: nil,- }-- // Only add to schema's table list if it's not a partition table- if !e.partitionTables[key] {- schema := e.getOrCreateSchema(schemaName)- schema.Tables = append(schema.Tables, table)- }- e.tables[key] = table-- return table-}--// Helper function to find materialized view in a schema-func (e *metadataExtractor) findMaterializedView(schemaName, viewName string) *storepb.MaterializedViewMetadata {- schema := e.getOrCreateSchema(schemaName)- if schema.MaterializedViews == nil {- return nil- }-- for _, mv := range schema.MaterializedViews {- if mv.Name == viewName {- return mv- }- }- return nil-}--// EnterCreateschemastmt is called when entering a create schema statement-func (e *metadataExtractor) EnterCreateschemastmt(ctx *parser.CreateschemastmtContext) {- if e.err != nil {- return- }-- // Try to get schema name directly from Colid first- if ctx.Colid() != nil {- schemaName := pgparser.NormalizePostgreSQLColid(ctx.Colid())- e.getOrCreateSchema(schemaName)- } else if ctx.Optschemaname() != nil && ctx.Optschemaname().Colid() != nil {- schemaName := pgparser.NormalizePostgreSQLColid(ctx.Optschemaname().Colid())- e.getOrCreateSchema(schemaName)- }-}--// EnterCreatestmt is called when entering a create table statement-func (e *metadataExtractor) EnterCreatestmt(ctx *parser.CreatestmtContext) {- if e.err != nil {- return- }-- if ctx.Qualified_name(0) == nil {- return- }-- qualifiedName := ctx.Qualified_name(0)- schemaName, tableName := e.extractSchemaAndTableName(qualifiedName)-- // Check if this is a partition table (CREATE TABLE ... PARTITION OF ...)- // We'll check if there's a second qualified_name which would be the parent table- if ctx.Qualified_name(1) != nil && ctx.PARTITION() != nil && ctx.OF() != nil {- // Mark this table as a partition- key := tableKey{schema: schemaName, table: tableName}- e.partitionTables[key] = true-- // Get the parent table and add this partition to it- parentSchema, parentTable := e.extractSchemaAndTableName(ctx.Qualified_name(1))- parentTableMetadata := e.getOrCreateTable(parentSchema, parentTable)-- // Create partition metadata- partition := &storepb.TablePartitionMetadata{- Name: tableName,- }-- // Extract FOR VALUES clause if present- if ctx.Partitionboundspec() != nil {- partition.Value = e.extractPartitionBoundSpec(ctx.Partitionboundspec())- }-- // Get the parent table's partition expression and type- parentKey := tableKey{schema: parentSchema, table: parentTable}- if expr, ok := e.partitionExpressions[parentKey]; ok {- partition.Expression = expr- }- if partitionType, ok := e.partitionTypes[parentKey]; ok {- partition.Type = partitionType- }-- if parentTableMetadata.Partitions == nil {- parentTableMetadata.Partitions = []*storepb.TablePartitionMetadata{}- }- parentTableMetadata.Partitions = append(parentTableMetadata.Partitions, partition)-- // Don't continue processing this table as a regular table- return- }-- // Create the table (it's not a partition table)- tableMetadata := e.getOrCreateTable(schemaName, tableName)-- // Extract table elements (columns, constraints)- if tableElementList := ctx.Opttableelementlist(); tableElementList != nil {- e.extractTableElements(tableElementList, tableMetadata, schemaName)- }-- // Extract partition info and store the expression and type- if partitionSpec := ctx.Optpartitionspec(); partitionSpec != nil {- // Store the partition expression and type for this table- key := tableKey{schema: schemaName, table: tableName}- e.partitionExpressions[key] = e.extractPartitionExpression(partitionSpec)- e.partitionTypes[key] = e.extractPartitionType(partitionSpec)- }-}--// extractSchemaAndTableName extracts schema and table name from qualified name-func (e *metadataExtractor) extractSchemaAndTableName(ctx parser.IQualified_nameContext) (string, string) {- if ctx == nil {- return e.currentSchema, ""- }-- // Use NormalizePostgreSQLQualifiedName which properly handles quoted identifiers- // This function removes quotes and handles case sensitivity correctly- parts := pgparser.NormalizePostgreSQLQualifiedName(ctx)- if len(parts) == 1 {- return e.currentSchema, parts[0]- } else if len(parts) == 2 {- return parts[0], parts[1]- }- return e.currentSchema, ""-}--// extractTableElements extracts columns and constraints from table elements-func (e *metadataExtractor) extractTableElements(ctx parser.IOpttableelementlistContext, table *storepb.TableMetadata, schemaName string) {- if ctx == nil {- return- }-- // Get the table element list- tableElementList := ctx.Tableelementlist()- if tableElementList == nil {- return- }-- // Process all table elements- for _, tableElement := range tableElementList.AllTableelement() {- if tableElement == nil {- continue- }-- // Handle column definitions- if columnDef := tableElement.ColumnDef(); columnDef != nil {- e.extractColumnDef(columnDef, table, schemaName)- }-- // Handle table constraints- if tableConstraint := tableElement.Tableconstraint(); tableConstraint != nil {- e.extractTableConstraint(tableConstraint, table, schemaName)- }- }-}--// extractColumnDef extracts column definition-func (e *metadataExtractor) extractColumnDef(ctx parser.IColumnDefContext, table *storepb.TableMetadata, schemaName string) {- if ctx == nil {- return- }-- column := &storepb.ColumnMetadata{- Nullable: true, // Default to nullable- }-- // Extract column name- if ctx.Colid() != nil {- column.Name = pgparser.NormalizePostgreSQLColid(ctx.Colid())- }-- // Extract data type- if ctx.Typename() != nil {- // Get the raw type name to check for SERIAL- rawTypeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Typename())- rawTypeName = strings.ToLower(rawTypeName)-- // Handle SERIAL types completely - PostgreSQL expands SERIAL into integer + sequence + default- switch rawTypeName {- case "serial", "serial4":- column.Nullable = false- column.Type = "integer"- // Create sequence and set default value- e.createSerialSequenceAndDefault(column, table, schemaName)- case "bigserial", "serial8":- column.Nullable = false- column.Type = "bigint"- // Create sequence and set default value- e.createSerialSequenceAndDefault(column, table, schemaName)- case "smallserial", "serial2":- column.Nullable = false- column.Type = "smallint"- // Create sequence and set default value- e.createSerialSequenceAndDefault(column, table, schemaName)- default:- column.Type = e.extractTypeNameWithSchema(ctx.Typename(), schemaName)- }- }-- // Extract column constraints- if colquallist := ctx.Colquallist(); colquallist != nil {- for _, colConstraint := range colquallist.AllColconstraint() {- if colConstraint != nil && colConstraint.Colconstraintelem() != nil {- // Extract constraint name if present- var constraintName string- if colConstraint.CONSTRAINT() != nil && colConstraint.Name() != nil {- constraintName = pgparser.NormalizePostgreSQLName(colConstraint.Name())- }- e.extractColumnConstraint(colConstraint.Colconstraintelem(), column, table, constraintName, schemaName)- }- }- }-- table.Columns = append(table.Columns, column)-}--// createSerialSequenceAndDefault creates a sequence for SERIAL columns and sets the default value-func (e *metadataExtractor) createSerialSequenceAndDefault(column *storepb.ColumnMetadata, table *storepb.TableMetadata, schemaName string) {- // Generate sequence name following PostgreSQL convention: tablename_columnname_seq- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)-- // Determine sequence data type based on column type- // SERIAL (integer) → bigint sequence (PostgreSQL standard)- // BIGSERIAL (bigint) → bigint sequence- // SMALLSERIAL (smallint) → bigint sequence- // All sequences in PostgreSQL are bigint by default- dataType := "bigint"-- // Create the sequence metadata with proper defaults matching PostgreSQL behavior- sequence := &storepb.SequenceMetadata{- Name: sequenceName,- DataType: dataType,- Start: "1",- Increment: "1",- MinValue: "1",- MaxValue: "9223372036854775807", // bigint max value- Cycle: false,- CacheSize: "1",- OwnerTable: table.Name,- OwnerColumn: column.Name,- }-- // Add sequence to the global sequences map using fully qualified name- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)- e.sequences[sequenceKey] = sequence-- // Set the column default value to use nextval()- // PostgreSQL stores this as: nextval('schema.sequence_name'::regclass)- column.Default = fmt.Sprintf("nextval('%s.%s'::regclass)", schemaName, sequenceName)-}--// extractTypeName extracts the type name from typename context-func extractTypeName(ctx parser.ITypenameContext) string {- if ctx == nil {- return ""- }- // Get the full text representation of the type- typeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- // Convert to lowercase to match sync.go output- typeName = strings.ToLower(typeName)-- return NormalizePostgreSQLType(typeName)-}--// getPostgreSQLArrayTypeName maps base types to their PostgreSQL internal array type names-// Based on PostgreSQL system catalog pg_type where array types start with underscore-func getPostgreSQLArrayTypeName(baseType string) string {- // Normalize the base type first to handle aliases- normalizedBase := strings.ToLower(strings.TrimSpace(baseType))-- // Map of base types to their PostgreSQL internal array type names- arrayTypeMap := map[string]string{- // Integer types- "int": "_int4",- "int4": "_int4",- "integer": "_int4",- "int2": "_int2",- "smallint": "_int2",- "int8": "_int8",- "bigint": "_int8",-- // Floating point types- "real": "_float4",- "float4": "_float4",- "double precision": "_float8",- "float8": "_float8",-- // Character types- "text": "_text",- "varchar": "_varchar",- "character varying": "_varchar",- "char": "_bpchar", // PostgreSQL uses bpchar internally for char- "character": "_bpchar", // PostgreSQL uses bpchar internally for character- "bpchar": "_bpchar",- "name": "_name",-- // Boolean type- "bool": "_bool",- "boolean": "_bool",-- // Numeric types- "numeric": "_numeric",- "decimal": "_numeric",-- // Date/time types- "date": "_date",- "time": "_time",- "time without time zone": "_time",- "time with time zone": "_timetz",- "timetz": "_timetz",- "timestamp": "_timestamp",- "timestamp without time zone": "_timestamp",- "timestamp with time zone": "_timestamptz",- "timestamptz": "_timestamptz",- "interval": "_interval",-- // Binary and other types- "bytea": "_bytea",- "uuid": "_uuid",- "json": "_json",- "jsonb": "_jsonb",- "xml": "_xml",- "money": "_money",- "inet": "_inet",- "cidr": "_cidr",- "macaddr": "_macaddr",- "macaddr8": "_macaddr8",-- // Geometric types- "point": "_point",- "line": "_line",- "lseg": "_lseg",- "box": "_box",- "path": "_path",- "polygon": "_polygon",- "circle": "_circle",-- // Full-text search types- "tsvector": "_tsvector",- "tsquery": "_tsquery",-- // Range types- "int4range": "_int4range",- "int8range": "_int8range",- "numrange": "_numrange",- "tsrange": "_tsrange",- "tstzrange": "_tstzrange",- "daterange": "_daterange",-- // Multi-range types (PostgreSQL 14+)- "int4multirange": "_int4multirange",- "int8multirange": "_int8multirange",- "nummultirange": "_nummultirange",- "tsmultirange": "_tsmultirange",- "tstzmultirange": "_tstzmultirange",- "datemultirange": "_datemultirange",-- // Bit string types- "bit": "_bit",- "bit varying": "_varbit",- "varbit": "_varbit",-- // Object identifier types- "oid": "_oid",- "regproc": "_regproc",- "regprocedure": "_regprocedure",- "regoper": "_regoper",- "regoperator": "_regoperator",- "regclass": "_regclass",- "regtype": "_regtype",- "regconfig": "_regconfig",- "regdictionary": "_regdictionary",- "regnamespace": "_regnamespace",- "regrole": "_regrole",- "regcollation": "_regcollation",-- // Other system types- "tid": "_tid",- "xid": "_xid",- "xid8": "_xid8",- "cid": "_cid",- "pg_lsn": "_pg_lsn",- "record": "_record",- "cstring": "_cstring",- "refcursor": "_refcursor",- "jsonpath": "_jsonpath",- "txid_snapshot": "_txid_snapshot",- "pg_snapshot": "_pg_snapshot",- "gtsvector": "_gtsvector",- "aclitem": "_aclitem",- "int2vector": "_int2vector",- "oidvector": "_oidvector",- }-- // Handle types with precision/scale (e.g., varchar(255), numeric(10,2), character varying(255))- if idx := strings.Index(normalizedBase, "("); idx != -1 {- baseTypeWithoutPrecision := normalizedBase[:idx]- if arrayType, exists := arrayTypeMap[baseTypeWithoutPrecision]; exists {- return arrayType- }- }-- // Direct lookup- if arrayType, exists := arrayTypeMap[normalizedBase]; exists {- return arrayType- }-- // For serial types, they should not have array equivalents in DDL,- // but if they do, map to their underlying integer array types- switch normalizedBase {- case "serial", "serial4":- return "_int4"- case "bigserial", "serial8":- return "_int8"- case "smallserial", "serial2":- return "_int2"- default:- // For non-serial types, proceed with fallback logic- }-- // Return empty string if no mapping found - caller will use fallback logic- return ""-}--// normalizePostgreSQLType normalizes PostgreSQL type names to match sync.go output-func NormalizePostgreSQLType(typeName string) string {- // Remove extra whitespace- typeName = strings.TrimSpace(typeName)-- // Handle array types FIRST before any other conversions- // text[] -> _text, int[] -> _int4, varchar(255)[] -> _varchar, etc.- // PostgreSQL treats multi-dimensional arrays the same as single dimension- if strings.HasSuffix(typeName, "[]") {- // Remove all array dimensions (int[][], int[] both become int)- baseType := typeName- for strings.HasSuffix(baseType, "[]") {- baseType = baseType[:len(baseType)-2]- }-- // Map base type to PostgreSQL's internal array type name- arrayType := getPostgreSQLArrayTypeName(baseType)- if arrayType != "" {- return arrayType- }-- // For unknown types, use the old logic as fallback- normalizedBase := NormalizePostgreSQLType(baseType)- return "_" + normalizedBase- }-- // Convert common type variations to match sync.go output- // varchar(n) -> character varying(n)- if strings.HasPrefix(typeName, "varchar(") {- return "character varying" + typeName[7:]- }- if typeName == "varchar" {- return "character varying"- }-- // SERIAL types are stored as integer/bigint in the catalog- if typeName == "serial" || typeName == "serial4" {- return "integer"- }- if typeName == "bigserial" || typeName == "serial8" {- return "bigint"- }- if typeName == "smallserial" || typeName == "serial2" {- return "smallint"- }-- // Handle timestamp without explicit timezone- if typeName == "timestamp" {- return "timestamp(6) without time zone" // PostgreSQL default precision is 6- }- // Handle timestamp with time zone (no precision specified)- if typeName == "timestamp with time zone" {- return "timestamp(6) with time zone" // PostgreSQL default precision is 6- }- // Handle timestamp with precision (preserve precision but add timezone info)- if strings.HasPrefix(typeName, "timestamp(") && strings.HasSuffix(typeName, ")") {- // Extract precision part: timestamp(3) -> (3)- precision := typeName[9:] // Get "(3)" part- return "timestamp" + precision + " without time zone"- }- // Handle time without explicit timezone (PostgreSQL default)- if typeName == "time" {- return "time(6) without time zone" // PostgreSQL default precision is 6- }- // Handle time with time zone (no precision specified)- if typeName == "time with time zone" {- return "time(6) with time zone" // PostgreSQL default precision is 6- }- // Handle time with precision (preserve precision but add timezone info)- if strings.HasPrefix(typeName, "time(") && strings.HasSuffix(typeName, ")") {- // Extract precision part: time(6) -> (6)- precision := typeName[4:] // Get "(6)" part- return "time" + precision + " without time zone"- }-- // Handle decimal -> numeric (preserve precision information)- if strings.HasPrefix(typeName, "decimal(") {- // Convert decimal(p,s) to numeric(p,s) while preserving precision- // Also normalize spacing to match sync.go format (remove spaces around commas)- precisionPart := typeName[7:] // Get "(p, s)" or "(p,s)" part- precisionPart = strings.ReplaceAll(precisionPart, " ", "") // Remove all spaces to match sync.go format- return "numeric" + precisionPart- }- if typeName == "decimal" {- return "numeric"- }-- // Normalize common type aliases to match PostgreSQL catalog output- switch typeName {- case "int", "int4":- return "integer"- case "int2":- return "smallint"- case "int8":- return "bigint"- case "float4":- return "real"- case "float8":- return "double precision"- case "bool":- return "boolean"- case "char":- return "character"- case "timestamptz":- return "timestamp(6) with time zone" // PostgreSQL default precision is 6- case "timetz":- return "time(6) with time zone" // PostgreSQL default precision is 6- case "varbit":- return "bit varying"- default:- // Return type as-is for unrecognized types- }-- // Handle specific length specifications for character types- if strings.HasPrefix(typeName, "char(") {- return "character" + typeName[4:]- }-- // Handle varbit with precision: varbit(n) -> bit varying(n)- if strings.HasPrefix(typeName, "varbit(") {- return "bit varying" + typeName[6:] // Replace "varbit" with "bit varying"- }-- // For all remaining types with precision/scale parameters, normalize spacing to match sync.go format- // This handles cases like "numeric(10, 2)" -> "numeric(10,2)" to match database sync output- if strings.Contains(typeName, "(") && strings.Contains(typeName, ")") {- // Remove spaces around commas in type specifications to match sync.go format- typeName = strings.ReplaceAll(typeName, ", ", ",")- typeName = strings.ReplaceAll(typeName, " ,", ",")- }-- return typeName-}--// extractTypeNameWithSchema extracts the type name and adds schema prefix for custom types-func (*metadataExtractor) extractTypeNameWithSchema(ctx parser.ITypenameContext, schemaName string) string {- if ctx == nil {- return ""- }-- // Get the raw type name first (without normalization)- rawTypeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- rawTypeName = strings.ToLower(strings.TrimSpace(rawTypeName))-- // Check if this is a built-in PostgreSQL type or an array type before normalization- // Array types should always go through normalization regardless of their base type- if isBuiltInType(rawTypeName) || isBuiltInType(NormalizePostgreSQLType(rawTypeName)) || strings.HasSuffix(rawTypeName, "[]") {- // For built-in types and array types, return the normalized version- return NormalizePostgreSQLType(rawTypeName)- }-- // Check if the type already has a schema prefix- if strings.Contains(rawTypeName, ".") {- // It's already schema-qualified, just normalize it- return NormalizePostgreSQLType(rawTypeName)- }-- // For custom types (like enums), add the schema prefix to the raw type name- return fmt.Sprintf("%s.%s", schemaName, rawTypeName)-}--// isBuiltInType checks if a type is a built-in PostgreSQL type-func isBuiltInType(typeName string) bool {- // Handle array types first (before removing precision info)- if strings.HasSuffix(typeName, "[]") {- // Remove all array dimensions (int[][], int[] both should be recognized)- arrayBase := typeName- for strings.HasSuffix(arrayBase, "[]") {- arrayBase = arrayBase[:len(arrayBase)-2]- }- // Recursively check if the base type is built-in- return isBuiltInType(arrayBase)- }- if strings.HasPrefix(typeName, "_") {- // PostgreSQL internal array notation _typename- arrayBase := typeName[1:]- return isBuiltInType(arrayBase)- }-- // Remove any precision/scale information for checking- baseType := typeName- if idx := strings.Index(typeName, "("); idx != -1 {- baseType = typeName[:idx]- }-- // Common built-in PostgreSQL types including both aliases and canonical forms- builtInTypes := map[string]bool{- // Integer types- "integer": true, "int": true, "int4": true,- "bigint": true, "int8": true,- "smallint": true, "int2": true,- "serial": true, "serial4": true,- "bigserial": true, "serial8": true, "smallserial": true, "serial2": true,-- // Numeric types- "numeric": true, "decimal": true,- "real": true, "float4": true,- "double precision": true, "float8": true,- "money": true,-- // Character types- "character": true, "char": true,- "character varying": true, "varchar": true,- "text": true,- "name": true,-- // Boolean- "boolean": true, "bool": true,-- // Date/time types- "date": true,- "time": true, "time without time zone": true, "time with time zone": true, "timetz": true,- "timestamp": true, "timestamp without time zone": true, "timestamp with time zone": true, "timestamptz": true,- "interval": true,-- // UUID- "uuid": true,-- // JSON types- "json": true, "jsonb": true,-- // XML- "xml": true,-- // Binary data- "bytea": true,-- // Bit string types- "bit": true, "bit varying": true, "varbit": true,-- // Network address types- "inet": true, "cidr": true, "macaddr": true, "macaddr8": true,-- // Geometric types- "point": true, "line": true, "lseg": true, "box": true,- "path": true, "polygon": true, "circle": true,-- // Full-text search types- "tsquery": true, "tsvector": true,-- // Range types- "int4range": true, "int8range": true, "numrange": true,- "tsrange": true, "tstzrange": true, "daterange": true,-- // Other types- "pg_lsn": true,- "oid": true, "regclass": true, "regproc": true, "regtype": true, "regoper": true,- "regoperator": true, "regconfig": true, "regdictionary": true,- "tid": true, "xid": true, "cid": true,- }-- return builtInTypes[baseType]-}--// extractColumnConstraint extracts column-level constraints-func (e *metadataExtractor) extractColumnConstraint(ctx parser.IColconstraintelemContext, column *storepb.ColumnMetadata, table *storepb.TableMetadata, constraintName string, schemaName string) {- if ctx == nil {- return- }-- switch {- case ctx.NOT() != nil && ctx.NULL_P() != nil:- column.Nullable = false- case ctx.NULL_P() != nil && ctx.NOT() == nil:- column.Nullable = true- case ctx.DEFAULT() != nil:- if expr := ctx.B_expr(); expr != nil {- rawDefault := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)- column.Default = e.normalizeDefaultValue(rawDefault, column, e.currentSchema)- }- case ctx.PRIMARY() != nil && ctx.KEY() != nil:- column.Nullable = false- // Create primary key index- index := &storepb.IndexMetadata{- Primary: true,- Unique: true,- IsConstraint: true,- Expressions: []string{column.Name},- Descending: []bool{false}, // Single column, ascending by default- Type: "btree",- Visible: false, // Match PostgreSQL database behavior- // Don't set KeyLength - PostgreSQL database doesn't return this information- }- // Use provided constraint name or generate one- if constraintName != "" {- index.Name = constraintName- } else {- index.Name = fmt.Sprintf("%s_pkey", table.Name)- }- // Generate definition for primary key index- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)- table.Indexes = append(table.Indexes, index)- case ctx.UNIQUE() != nil:- // Create unique index- index := &storepb.IndexMetadata{- Unique: true,- IsConstraint: true,- Expressions: []string{column.Name},- Descending: []bool{false}, // Single column, ascending by default- Type: "btree",- Visible: false, // Match PostgreSQL database behavior- // Don't set KeyLength - PostgreSQL database doesn't return this information- }- // Use provided constraint name or generate one- if constraintName != "" {- index.Name = constraintName- } else {- index.Name = fmt.Sprintf("%s_%s_key", table.Name, column.Name)- }- // Generate definition for unique index- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)- table.Indexes = append(table.Indexes, index)- case ctx.REFERENCES() != nil:- // Foreign key constraint- fk := &storepb.ForeignKeyMetadata{- Columns: []string{column.Name},- }- if constraintName != "" {- fk.Name = constraintName- }- if qualifiedName := ctx.Qualified_name(); qualifiedName != nil {- refSchema, refTable := e.extractSchemaAndTableName(qualifiedName)- fk.ReferencedSchema = refSchema- fk.ReferencedTable = refTable- }- if optColumnList := ctx.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {- fk.ReferencedColumns = extractColumnList(optColumnList.Columnlist())- }- // Extract ON DELETE/UPDATE actions- fk.OnDelete = "NO ACTION" // Default- fk.OnUpdate = "NO ACTION" // Default- if keyActions := ctx.Key_actions(); keyActions != nil {- if keyDelete := keyActions.Key_delete(); keyDelete != nil {- fk.OnDelete = extractKeyAction(keyDelete)- }- if keyUpdate := keyActions.Key_update(); keyUpdate != nil {- fk.OnUpdate = extractKeyActionUpdate(keyUpdate)- }- }- // Extract MATCH type- fk.MatchType = extractMatchType(ctx)-- // Generate constraint name if not provided (PostgreSQL auto-generates names)- if fk.Name == "" {- // For column-level foreign keys, use {table}_{column}_fkey format- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, column.Name)- }- // TODO: Generate definition for foreign key (ForeignKeyMetadata might not have Definition field)- // fk.Definition = e.generateForeignKeyDefinition(fk, table.Name, schemaName)-- if table.ForeignKeys == nil {- table.ForeignKeys = []*storepb.ForeignKeyMetadata{}- }- table.ForeignKeys = append(table.ForeignKeys, fk)- case ctx.GENERATED() != nil && ctx.IDENTITY_P() != nil:- // Handle GENERATED ALWAYS AS IDENTITY or GENERATED BY DEFAULT AS IDENTITY- if generatedWhen := ctx.Generated_when(); generatedWhen != nil {- if generatedWhen.ALWAYS() != nil {- column.IdentityGeneration = storepb.ColumnMetadata_ALWAYS- } else if generatedWhen.BY() != nil && generatedWhen.DEFAULT() != nil {- column.IdentityGeneration = storepb.ColumnMetadata_BY_DEFAULT- }- }-- // Create identity sequence for this column- e.createIdentitySequence(table, column, schemaName)- case ctx.CHECK() != nil:- // Column-level CHECK constraint- check := &storepb.CheckConstraintMetadata{}-- // Generate constraint name if not provided (PostgreSQL auto-generates names)- if constraintName != "" {- check.Name = constraintName- } else {- // For column-level check constraints, use {table}_{column}_check format- check.Name = fmt.Sprintf("%s_%s_check", table.Name, column.Name)- }-- // Extract the check expression- if expr := ctx.A_expr(); expr != nil {- rawExpression := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)- check.Expression = strings.TrimSpace(rawExpression)- }-- // Add to table's check constraints- if table.CheckConstraints == nil {- table.CheckConstraints = []*storepb.CheckConstraintMetadata{}- }- table.CheckConstraints = append(table.CheckConstraints, check)- default:- // Ignore other column constraints- }-}--// extractTableConstraint extracts table-level constraints-func (e *metadataExtractor) extractTableConstraint(ctx parser.ITableconstraintContext, table *storepb.TableMetadata, schemaName string) {- if ctx == nil {- return- }-- constraintElem := ctx.Constraintelem()- if constraintElem == nil {- return- }-- // Get constraint name- var constraintName string- if ctx.Name() != nil {- constraintName = pgparser.NormalizePostgreSQLName(ctx.Name())- }-- switch {- case constraintElem.PRIMARY() != nil && constraintElem.KEY() != nil:- // Primary key constraint- index := &storepb.IndexMetadata{- Name: constraintName,- Primary: true,- Unique: true,- IsConstraint: true,- Expressions: []string{},- Descending: []bool{},- Type: "btree",- Visible: false, // Match PostgreSQL database behavior- // Don't set KeyLength - PostgreSQL database doesn't return this information- }- if index.Name == "" {- index.Name = fmt.Sprintf("%s_pkey", table.Name)- }- // Extract columns- // For PRIMARY KEY constraints, columns are in direct Columnlist, not Opt_column_list- if columnList := constraintElem.Columnlist(); columnList != nil {- columns := extractColumnList(columnList)- index.Expressions = columns- // Fill Descending array with false for all columns (default ascending)- for range columns {- index.Descending = append(index.Descending, false)- }- } else if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {- // Fallback to Opt_column_list if direct Columnlist is not available- columns := extractColumnList(optColumnList.Columnlist())- index.Expressions = columns- // Fill Descending array with false for all columns (default ascending)- for range columns {- index.Descending = append(index.Descending, false)- }- }- // Generate definition for primary key constraint index- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)- table.Indexes = append(table.Indexes, index)-- case constraintElem.UNIQUE() != nil:- // Unique constraint- index := &storepb.IndexMetadata{- Name: constraintName,- Unique: true,- IsConstraint: true,- Expressions: []string{},- Descending: []bool{},- Type: "btree",- Visible: false, // Match PostgreSQL database behavior- // Don't set KeyLength - PostgreSQL database doesn't return this information- }- // Extract columns- // For UNIQUE constraints, columns are in direct Columnlist, not Opt_column_list- if columnList := constraintElem.Columnlist(); columnList != nil {- columns := extractColumnList(columnList)- index.Expressions = columns- // Fill Descending array with false for all columns (default ascending)- for range columns {- index.Descending = append(index.Descending, false)- }- } else if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {- // Fallback to Opt_column_list if direct Columnlist is not available- columns := extractColumnList(optColumnList.Columnlist())- index.Expressions = columns- // Fill Descending array with false for all columns (default ascending)- for range columns {- index.Descending = append(index.Descending, false)- }- }- // Generate constraint name if not provided- if index.Name == "" && len(index.Expressions) > 0 {- // PostgreSQL automatically generates constraint names in the format: table_column_key- // For multi-column constraints: table_column1_column2_..._key- index.Name = fmt.Sprintf("%s_%s_key", table.Name, strings.Join(index.Expressions, "_"))- }- // Generate definition for unique constraint index- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)- table.Indexes = append(table.Indexes, index)-- case constraintElem.CHECK() != nil:- // Check constraint- check := &storepb.CheckConstraintMetadata{}- // Generate constraint name if not provided (PostgreSQL auto-generates names)- if constraintName != "" {- check.Name = constraintName- } else {- // For table-level check constraints, use {table}_check_{index} format- checkIndex := len(table.CheckConstraints) + 1- check.Name = fmt.Sprintf("%s_check_%d", table.Name, checkIndex)- }- if expr := constraintElem.A_expr(); expr != nil {- rawExpression := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)- check.Expression = strings.TrimSpace(rawExpression)- }- if table.CheckConstraints == nil {- table.CheckConstraints = []*storepb.CheckConstraintMetadata{}- }- table.CheckConstraints = append(table.CheckConstraints, check)-- case constraintElem.FOREIGN() != nil && constraintElem.KEY() != nil:- // Foreign key constraint- fk := &storepb.ForeignKeyMetadata{- Name: constraintName,- Columns: []string{},- ReferencedColumns: []string{},- }- // Extract local columns from Columnlist (before REFERENCES)- if columnList := constraintElem.Columnlist(); columnList != nil {- fk.Columns = extractColumnList(columnList)- }- // Extract referenced table- if qualifiedName := constraintElem.Qualified_name(); qualifiedName != nil {- refSchema, refTable := e.extractSchemaAndTableName(qualifiedName)- fk.ReferencedSchema = refSchema- fk.ReferencedTable = refTable- }- // Extract referenced columns from Opt_column_list (after REFERENCES table_name)- if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {- fk.ReferencedColumns = extractColumnList(optColumnList.Columnlist())- } else if len(fk.Columns) > 0 {- // If referenced columns not specified, assume they match the local columns- fk.ReferencedColumns = make([]string, len(fk.Columns))- copy(fk.ReferencedColumns, fk.Columns)- }- // Extract ON DELETE/UPDATE actions- fk.OnDelete = "NO ACTION" // Default- fk.OnUpdate = "NO ACTION" // Default- if keyActions := constraintElem.Key_actions(); keyActions != nil {- if keyDelete := keyActions.Key_delete(); keyDelete != nil {- fk.OnDelete = extractKeyAction(keyDelete)- }- if keyUpdate := keyActions.Key_update(); keyUpdate != nil {- fk.OnUpdate = extractKeyActionUpdate(keyUpdate)- }- }- // Extract MATCH type- fk.MatchType = extractMatchTypeFromConstraintElem(constraintElem)-- // Generate constraint name if not provided (PostgreSQL auto-generates names)- if fk.Name == "" {- // PostgreSQL typically uses {table}_{column}_fkey format for single column foreign keys- if len(fk.Columns) == 1 {- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, fk.Columns[0])- } else if len(fk.Columns) > 1 {- // For multi-column foreign keys, use first column name- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, fk.Columns[0])- } else {- // Fallback name- fk.Name = fmt.Sprintf("%s_fkey", table.Name)- }- }-- if table.ForeignKeys == nil {- table.ForeignKeys = []*storepb.ForeignKeyMetadata{}- }- table.ForeignKeys = append(table.ForeignKeys, fk)-- case constraintElem.EXCLUDE() != nil:- // EXCLUDE constraint- exclude := &storepb.ExcludeConstraintMetadata{}- // Set constraint name- if constraintName != "" {- exclude.Name = constraintName- } else {- // Generate name if not provided (PostgreSQL auto-generates names)- excludeIndex := len(table.ExcludeConstraints) + 1- exclude.Name = fmt.Sprintf("%s_exclude_%d", table.Name, excludeIndex)- }-- // Extract full EXCLUDE constraint expression from token stream- // This includes "EXCLUDE USING gist (...)" format- if parser := ctx.GetParser(); parser != nil {- if tokenStream := parser.GetTokenStream(); tokenStream != nil {- start := constraintElem.GetStart()- stop := constraintElem.GetStop()- if start != nil && stop != nil {- rawExpression := tokenStream.GetTextFromTokens(start, stop)- exclude.Expression = strings.TrimSpace(rawExpression)- }- }- }-- if table.ExcludeConstraints == nil {- table.ExcludeConstraints = []*storepb.ExcludeConstraintMetadata{}- }- table.ExcludeConstraints = append(table.ExcludeConstraints, exclude)-- default:- // Other constraint types not handled- }-}--// extractKeyAction extracts the foreign key delete action type-func extractKeyAction(ctx parser.IKey_deleteContext) string {- if ctx == nil {- return "NO ACTION"- }- keyAction := ctx.Key_action()- if keyAction == nil {- return "NO ACTION"- }- switch {- case keyAction.CASCADE() != nil:- return "CASCADE"- case keyAction.SET() != nil && keyAction.NULL_P() != nil:- return "SET NULL"- case keyAction.SET() != nil && keyAction.DEFAULT() != nil:- return "SET DEFAULT"- case keyAction.RESTRICT() != nil:- return "RESTRICT"- default:- return "NO ACTION"- }-}--// extractKeyActionUpdate extracts the foreign key update action type-func extractKeyActionUpdate(ctx parser.IKey_updateContext) string {- if ctx == nil {- return "NO ACTION"- }- keyAction := ctx.Key_action()- if keyAction == nil {- return "NO ACTION"- }- switch {- case keyAction.CASCADE() != nil:- return "CASCADE"- case keyAction.SET() != nil && keyAction.NULL_P() != nil:- return "SET NULL"- case keyAction.SET() != nil && keyAction.DEFAULT() != nil:- return "SET DEFAULT"- case keyAction.RESTRICT() != nil:- return "RESTRICT"- default:- return "NO ACTION"- }-}--// extractMatchType extracts the match type from column constraint context-func extractMatchType(ctx parser.IColconstraintelemContext) string {- if ctx == nil || ctx.Key_match() == nil {- return "SIMPLE" // PostgreSQL default is MATCH SIMPLE- }- keyMatch := ctx.Key_match()- switch {- case keyMatch.FULL() != nil:- return "FULL"- case keyMatch.PARTIAL() != nil:- return "PARTIAL"- case keyMatch.SIMPLE() != nil:- return "SIMPLE"- default:- return ""- }-}--// extractMatchTypeFromConstraintElem extracts the match type from table constraint context-func extractMatchTypeFromConstraintElem(ctx parser.IConstraintelemContext) string {- if ctx == nil || ctx.Key_match() == nil {- return "SIMPLE" // PostgreSQL default is MATCH SIMPLE- }- keyMatch := ctx.Key_match()- switch {- case keyMatch.FULL() != nil:- return "FULL"- case keyMatch.PARTIAL() != nil:- return "PARTIAL"- case keyMatch.SIMPLE() != nil:- return "SIMPLE"- default:- return ""- }-}--// extractColumnList extracts column names from columnlist-func extractColumnList(ctx parser.IColumnlistContext) []string {- if ctx == nil {- return nil- }-- var columns []string- for _, colElem := range ctx.AllColumnElem() {- if colElem.Colid() != nil {- columns = append(columns, pgparser.NormalizePostgreSQLColid(colElem.Colid()))- }- }- return columns-}--// extractPartitionExpression extracts the partition expression (e.g., "RANGE (sale_date)")-func (*metadataExtractor) extractPartitionExpression(ctx parser.IOptpartitionspecContext) string {- if ctx == nil {- return ""- }-- partitionSpec := ctx.Partitionspec()- if partitionSpec == nil {- return ""- }-- // Extract the full partition expression including the method (e.g., "RANGE (sale_date)")- expression := ctx.GetParser().GetTokenStream().GetTextFromTokens(partitionSpec.Colid().GetStart(), partitionSpec.CLOSE_PAREN().GetSymbol())- return expression-}--// extractPartitionType extracts the partition type from the partition specification-func (*metadataExtractor) extractPartitionType(ctx parser.IOptpartitionspecContext) storepb.TablePartitionMetadata_Type {- if ctx == nil {- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED- }-- partitionSpec := ctx.Partitionspec()- if partitionSpec == nil {- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED- }-- // Extract the partition method from the full text- fullText := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(partitionSpec)- upperText := strings.ToUpper(fullText)-- // Determine partition type from the method keyword after "PARTITION BY"- if strings.Contains(upperText, "PARTITION BY RANGE") {- return storepb.TablePartitionMetadata_RANGE- } else if strings.Contains(upperText, "PARTITION BY LIST") {- return storepb.TablePartitionMetadata_LIST- } else if strings.Contains(upperText, "PARTITION BY HASH") {- return storepb.TablePartitionMetadata_HASH- }-- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED-}--// EnterAltertablestmt is called when entering an ALTER TABLE statement-func (e *metadataExtractor) EnterAltertablestmt(ctx *parser.AltertablestmtContext) {- if e.err != nil {- return- }-- // Check if this is an ATTACH PARTITION command- if partitionCmd := ctx.Partition_cmd(); partitionCmd != nil {- if partitionCmd.ATTACH() != nil && partitionCmd.PARTITION() != nil {- e.handleAttachPartition(ctx, partitionCmd)- }- return- }-- // Check if this is an ALTER TABLE command with alter_table_cmds- if alterTableCmdList := ctx.Alter_table_cmds(); alterTableCmdList != nil {- e.handleAlterTableCommands(ctx, alterTableCmdList)- }-}--// handleAlterTableCommands processes ALTER TABLE commands like ADD CONSTRAINT-func (e *metadataExtractor) handleAlterTableCommands(ctx *parser.AltertablestmtContext, alterTableCmdList parser.IAlter_table_cmdsContext) {- // Get the table name from the ALTER TABLE statement- if relationExpr := ctx.Relation_expr(); relationExpr != nil {- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {- schemaName, tableName := e.extractSchemaAndTableName(qualifiedName)-- // Get or create the table- table := e.getOrCreateTable(schemaName, tableName)-- // Process each alter table command- for _, alterTableCmd := range alterTableCmdList.AllAlter_table_cmd() {- if alterTableCmd == nil {- continue- }-- // Debug: check what type of command this is- hasAlter := alterTableCmd.ALTER() != nil- hasAdd := alterTableCmd.ADD_P() != nil- hasGenerated := alterTableCmd.GENERATED() != nil- hasIdentity := alterTableCmd.IDENTITY_P() != nil-- // Check if this is an ADD CONSTRAINT command- if alterTableCmd.ADD_P() != nil && alterTableCmd.Tableconstraint() != nil {- e.extractTableConstraint(alterTableCmd.Tableconstraint(), table, schemaName)- continue- }-- // Check if this is an ALTER COLUMN ADD GENERATED AS IDENTITY command- // According to g4: ALTER opt_column? colid ADD_P GENERATED generated_when AS IDENTITY_P- // The COLUMN keyword is optional (opt_column?)- if hasAlter && hasAdd && hasGenerated && hasIdentity {- e.handleAlterColumn(alterTableCmd, table, schemaName)- }- }- }- }-}--// handleAlterColumn processes ALTER TABLE ALTER COLUMN commands-func (e *metadataExtractor) handleAlterColumn(alterTableCmd parser.IAlter_table_cmdContext, table *storepb.TableMetadata, schemaName string) {- // Get the column name from colid (based on g4: ALTER opt_column? colid ADD_P GENERATED...)- columnName := ""- allColids := alterTableCmd.AllColid()- if len(allColids) > 0 {- // The column name is the first colid- columnName = pgparser.NormalizePostgreSQLColid(allColids[0])- }-- // Debug log- _ = columnName // TODO: remove after debug-- if columnName == "" {- return- }-- // Find the column in the table- var targetColumn *storepb.ColumnMetadata- for _, col := range table.Columns {- if col.Name == columnName {- targetColumn = col- break- }- }-- // If column doesn't exist, we might need to create it- // But for now, we'll just skip if the column doesn't exist- if targetColumn == nil {- return- }-- // Check if this is ADD GENERATED AS IDENTITY (from g4: ALTER opt_column? colid ADD_P GENERATED generated_when AS IDENTITY_P)- if alterTableCmd.ADD_P() != nil && alterTableCmd.GENERATED() != nil && alterTableCmd.IDENTITY_P() != nil {- // Set identity generation mode- if generatedWhen := alterTableCmd.Generated_when(); generatedWhen != nil {- if generatedWhen.ALWAYS() != nil {- targetColumn.IdentityGeneration = storepb.ColumnMetadata_ALWAYS- } else if generatedWhen.BY() != nil && generatedWhen.DEFAULT() != nil {- targetColumn.IdentityGeneration = storepb.ColumnMetadata_BY_DEFAULT- }- }-- // Get sequence options from AST- var seqOptList parser.ISeqoptlistContext- if optParenSeqList := alterTableCmd.Optparenthesizedseqoptlist(); optParenSeqList != nil {- seqOptList = optParenSeqList.Seqoptlist()- }-- // Create the identity sequence with options from AST- e.createIdentitySequenceFromAST(table, targetColumn, schemaName, seqOptList)- }-}--// handleAttachPartition processes ALTER TABLE ATTACH PARTITION statements-func (e *metadataExtractor) handleAttachPartition(ctx *parser.AltertablestmtContext, partitionCmd parser.IPartition_cmdContext) {- // Get the main table name- if relationExpr := ctx.Relation_expr(); relationExpr != nil {- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {- mainSchema, mainTable := e.extractSchemaAndTableName(qualifiedName)-- // Get the partition name- if partitionQualifiedName := partitionCmd.Qualified_name(); partitionQualifiedName != nil {- partitionSchema, partitionName := e.extractSchemaAndTableName(partitionQualifiedName)-- // Get or create the main table- mainTableMetadata := e.getOrCreateTable(mainSchema, mainTable)-- // Create partition metadata- partition := &storepb.TablePartitionMetadata{- Name: partitionName,- }-- // Extract the FOR VALUES clause if present- if partitionBound := partitionCmd.Partitionboundspec(); partitionBound != nil {- partition.Value = e.extractPartitionBoundSpec(partitionBound)- }-- // Add to main table's partitions- if mainTableMetadata.Partitions == nil {- mainTableMetadata.Partitions = []*storepb.TablePartitionMetadata{}- }- mainTableMetadata.Partitions = append(mainTableMetadata.Partitions, partition)-- // Ensure the partition table exists in our metadata- // This handles cases where the partition table was created separately- _ = e.getOrCreateTable(partitionSchema, partitionName)- }- }- }-}--// EnterCreatematviewstmt is called when entering a CREATE MATERIALIZED VIEW statement-func (e *metadataExtractor) EnterCreatematviewstmt(ctx *parser.CreatematviewstmtContext) {- if e.err != nil {- return- }-- // Get the materialized view target- mvTarget := ctx.Create_mv_target()- if mvTarget == nil {- return- }-- // Extract schema and view name from the target- var schemaName, viewName string- if qualifiedName := mvTarget.Qualified_name(); qualifiedName != nil {- schemaName, viewName = e.extractSchemaAndTableName(qualifiedName)- } else {- return- }-- schema := e.getOrCreateSchema(schemaName)-- // Create materialized view metadata- materializedView := &storepb.MaterializedViewMetadata{- Name: viewName,- }-- // Extract the view definition- if ctx.AS() != nil && ctx.Selectstmt() != nil {- // Get the full SELECT statement text- selectCtx := ctx.Selectstmt()- startToken := selectCtx.GetStart()- stopToken := selectCtx.GetStop()- if startToken != nil && stopToken != nil {- materializedView.Definition = ctx.GetParser().GetTokenStream().GetTextFromTokens(startToken, stopToken)- }- }-- // Add to schema's materialized views- if schema.MaterializedViews == nil {- schema.MaterializedViews = []*storepb.MaterializedViewMetadata{}- }- schema.MaterializedViews = append(schema.MaterializedViews, materializedView)-}--// extractPartitionBoundSpec extracts the partition bound specification (FOR VALUES clause)-func (*metadataExtractor) extractPartitionBoundSpec(ctx parser.IPartitionboundspecContext) string {- if ctx == nil {- return ""- }-- // Get the full text of the partition bound specification- return ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)-}--// EnterCreateseqstmt is called when entering a create sequence statement-func (e *metadataExtractor) EnterCreateseqstmt(ctx *parser.CreateseqstmtContext) {- if e.err != nil {- return- }-- if ctx.Qualified_name() == nil {- return- }-- schemaName, sequenceName := e.extractSchemaAndTableName(ctx.Qualified_name())- schemaMetadata := e.getOrCreateSchema(schemaName)-- sequence := &storepb.SequenceMetadata{- Name: sequenceName,- DataType: "bigint", // Default for PostgreSQL- Start: "1",- Increment: "1",- MinValue: "1",- MaxValue: "9223372036854775807",- Cycle: false,- CacheSize: "1",- }-- // Extract sequence options- if optSeqList := ctx.Optseqoptlist(); optSeqList != nil && optSeqList.Seqoptlist() != nil {- for _, seqOptElem := range optSeqList.Seqoptlist().AllSeqoptelem() {- if seqOptElem == nil {- continue- }- e.extractSequenceOption(seqOptElem, sequence)- }- }-- // Store the sequence temporarily for OWNED BY processing- e.sequences[fmt.Sprintf("%s.%s", schemaName, sequenceName)] = sequence-- if schemaMetadata.Sequences == nil {- schemaMetadata.Sequences = []*storepb.SequenceMetadata{}- }- schemaMetadata.Sequences = append(schemaMetadata.Sequences, sequence)-}--// EnterAlterseqstmt is called when entering an ALTER SEQUENCE statement-func (e *metadataExtractor) EnterAlterseqstmt(ctx *parser.AlterseqstmtContext) {- if e.err != nil {- return- }-- if ctx.Qualified_name() == nil {- return- }-- schemaName, sequenceName := e.extractSchemaAndTableName(ctx.Qualified_name())- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)-- // Find the existing sequence or create one if it doesn't exist- sequence, exists := e.sequences[sequenceKey]- if !exists {- // If sequence doesn't exist yet, create it with defaults- schemaMetadata := e.getOrCreateSchema(schemaName)- sequence = &storepb.SequenceMetadata{- Name: sequenceName,- DataType: "bigint",- Start: "1",- Increment: "1",- MinValue: "1",- MaxValue: "9223372036854775807",- Cycle: false,- CacheSize: "1",- }- e.sequences[sequenceKey] = sequence- if schemaMetadata.Sequences == nil {- schemaMetadata.Sequences = []*storepb.SequenceMetadata{}- }- schemaMetadata.Sequences = append(schemaMetadata.Sequences, sequence)- }-- // Process sequence options from ALTER SEQUENCE- if seqOptList := ctx.Seqoptlist(); seqOptList != nil {- for _, seqOptElem := range seqOptList.AllSeqoptelem() {- if seqOptElem == nil {- continue- }- e.extractSequenceOption(seqOptElem, sequence)- }- }-}--// EnterCommentstmt is called when entering a COMMENT statement-func (e *metadataExtractor) EnterCommentstmt(ctx *parser.CommentstmtContext) {- if e.err != nil {- return- }-- // Extract comment text (common for all comment types)- var comment string- if commentText := ctx.Comment_text(); commentText != nil {- if commentText.NULL_P() != nil {- // COMMENT ... IS NULL means remove comment- comment = ""- } else if commentText.Sconst() != nil {- // Extract the string constant, removing quotes- commentStr := commentText.GetText()- if len(commentStr) >= 2 && commentStr[0] == '\'' && commentStr[len(commentStr)-1] == '\'' {- // Remove quotes and unescape- comment = commentStr[1 : len(commentStr)-1]- // Unescape single quotes ('' -> ')- comment = strings.ReplaceAll(comment, "''", "'")- }- }- }-- // Handle COMMENT ON COLUMN- if ctx.COLUMN() != nil && ctx.Any_name() != nil {- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())- if len(parts) < 2 {- return- }-- var schemaName, tableName, columnName string- if len(parts) == 2 {- schemaName = e.currentSchema- tableName = parts[0]- columnName = parts[1]- } else if len(parts) == 3 {- schemaName = parts[0]- tableName = parts[1]- columnName = parts[2]- } else {- return- }-- schema := e.getOrCreateSchema(schemaName)- for _, table := range schema.Tables {- if table.Name == tableName {- for i, col := range table.Columns {- if col.Name == columnName {- table.Columns[i].Comment = comment- return- }- }- }- }- return- }-- // Handle COMMENT ON TABLE/SEQUENCE/VIEW etc (object_type_any_name)- if ctx.Object_type_any_name() != nil && ctx.Any_name() != nil {- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())- if len(parts) == 0 {- return- }-- var schemaName, objectName string- if len(parts) == 1 {- schemaName = e.currentSchema- objectName = parts[0]- } else if len(parts) >= 2 {- schemaName = parts[0]- objectName = parts[1]- }-- schema := e.getOrCreateSchema(schemaName)- objectType := ctx.Object_type_any_name()-- // COMMENT ON TABLE- if objectType.TABLE() != nil {- for _, table := range schema.Tables {- if table.Name == objectName {- table.Comment = comment- return- }- }- }-- // COMMENT ON SEQUENCE- if objectType.SEQUENCE() != nil {- for _, seq := range schema.Sequences {- if seq.Name == objectName {- seq.Comment = comment- return- }- }- }-- // COMMENT ON VIEW- if objectType.VIEW() != nil {- for _, view := range schema.Views {- if view.Name == objectName {- view.Comment = comment- return- }- }- }-- // COMMENT ON MATERIALIZED VIEW- if objectType.MATERIALIZED() != nil && objectType.VIEW() != nil {- for _, view := range schema.MaterializedViews {- if view.Name == objectName {- view.Comment = comment- return- }- }- }-- // COMMENT ON INDEX- if objectType.INDEX() != nil {- for _, table := range schema.Tables {- for i, index := range table.Indexes {- if index.Name == objectName {- table.Indexes[i].Comment = comment- return- }- }- }- }-- return- }-- // Handle COMMENT ON SCHEMA / EXTENSION / etc.- if ctx.Object_type_name() != nil && ctx.Name() != nil {- objectType := strings.ToUpper(ctx.Object_type_name().GetText())- name := pgparser.NormalizePostgreSQLName(ctx.Name())- switch objectType {- case "SCHEMA":- if name != "" {- schema := e.getOrCreateSchema(name)- schema.Comment = comment- }- case "EXTENSION":- if ext, exists := e.extensions[name]; exists {- ext.Description = comment- }- default:- }- return- }-- // Handle COMMENT ON FUNCTION/PROCEDURE- if (ctx.FUNCTION() != nil || ctx.PROCEDURE() != nil) && ctx.Function_with_argtypes() != nil {- funcWithArgs := ctx.Function_with_argtypes()- if funcWithArgs.Func_name() == nil {- return- }-- // Extract function name- parts := pgparser.NormalizePostgreSQLFuncName(funcWithArgs.Func_name())- if len(parts) == 0 {- return- }-- var schemaName, funcName string- if len(parts) == 1 {- schemaName = e.currentSchema- funcName = parts[0]- } else if len(parts) >= 2 {- schemaName = parts[0]- funcName = parts[1]- }-- schema := e.getOrCreateSchema(schemaName)- for i, function := range schema.Functions {- if function.Name == funcName {- schema.Functions[i].Comment = comment- return- }- }- return- }-- // Handle COMMENT ON TYPE (including ENUM types)- if ctx.TYPE_P() != nil && ctx.AllTypename() != nil && len(ctx.AllTypename()) > 0 {- // Extract type name from typename- typename := ctx.AllTypename()[0]- var schemaName, typeName string-- // Try to get qualified name from typename- if typename.Qualified_name() != nil {- schemaName, typeName = e.extractSchemaAndTableName(typename.Qualified_name())- } else if typename.Simpletypename() != nil {- // Simple type name - get the text representation- typeText := typename.GetParser().GetTokenStream().GetTextFromRuleContext(typename)- // typeText might be "myschema.status" or just "status"- // We need to split it if it contains a dot- parts := strings.Split(typeText, ".")- if len(parts) == 2 {- schemaName = parts[0]- typeName = parts[1]- } else {- typeName = typeText- schemaName = e.currentSchema- }- }-- if typeName != "" {- schema := e.getOrCreateSchema(schemaName)- for i, enumType := range schema.EnumTypes {- if enumType.Name == typeName {- schema.EnumTypes[i].Comment = comment- return- }- }- }- return- }-}--// EnterViewstmt is called when entering a create view statement-func (e *metadataExtractor) EnterViewstmt(ctx *parser.ViewstmtContext) {- if e.err != nil {- return- }-- if ctx.Qualified_name() == nil {- return- }-- schemaName, viewName := e.extractSchemaAndTableName(ctx.Qualified_name())- schemaMetadata := e.getOrCreateSchema(schemaName)-- // Create regular view metadata- // Note: Materialized views are not currently supported by the parser and are handled by sync- viewMetadata := &storepb.ViewMetadata{- Name: viewName,- }-- // Extract view definition- if ctx.Selectstmt() != nil {- viewMetadata.Definition = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Selectstmt())- }-- if schemaMetadata.Views == nil {- schemaMetadata.Views = []*storepb.ViewMetadata{}- }- schemaMetadata.Views = append(schemaMetadata.Views, viewMetadata)-}--// EnterCreatefunctionstmt is called when entering a create function statement-func (e *metadataExtractor) EnterCreatefunctionstmt(ctx *parser.CreatefunctionstmtContext) {- if e.err != nil {- return- }-- funcNameCtx := ctx.Func_name()- if funcNameCtx == nil {- return- }-- // Use NormalizePostgreSQLFuncName which properly handles quoted identifiers- parts := pgparser.NormalizePostgreSQLFuncName(funcNameCtx)- if len(parts) == 0 {- return- }-- var schemaName, funcName string- if len(parts) == 1 {- schemaName = e.currentSchema- funcName = parts[0]- } else if len(parts) >= 2 {- schemaName = parts[0]- funcName = parts[1]- }-- if funcName == "" {- return- }-- schemaMetadata := e.getOrCreateSchema(schemaName)-- functionMetadata := &storepb.FunctionMetadata{- Name: funcName,- Definition: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- Signature: ExtractFunctionSignature(ctx, funcName),- }-- if schemaMetadata.Functions == nil {- schemaMetadata.Functions = []*storepb.FunctionMetadata{}- }- schemaMetadata.Functions = append(schemaMetadata.Functions, functionMetadata)-}--// ExtractFunctionSignature extracts the function signature with parameter types-func ExtractFunctionSignature(ctx *parser.CreatefunctionstmtContext, funcName string) string {- var signature strings.Builder- signature.WriteString(funcName)- signature.WriteString(`(`)-- if funcArgs := ctx.Func_args_with_defaults(); funcArgs != nil {- if funcArgsList := funcArgs.Func_args_with_defaults_list(); funcArgsList != nil {- args := funcArgsList.AllFunc_arg_with_default()- for i, arg := range args {- if i > 0 {- signature.WriteString(", ")- }-- if funcArg := arg.Func_arg(); funcArg != nil {- // Extract parameter name if present- if paramName := funcArg.Param_name(); paramName != nil {- // Param_name returns the parameter name directly as text- signature.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(paramName))- signature.WriteString(" ")- }-- // Extract argument class (IN/OUT/INOUT/VARIADIC)- if argClass := funcArg.Arg_class(); argClass != nil {- if argClass.OUT_P() != nil {- signature.WriteString("OUT ")- } else if argClass.INOUT() != nil {- signature.WriteString("INOUT ")- } else if argClass.VARIADIC() != nil {- signature.WriteString("VARIADIC ")- }- // IN is default and usually omitted- }-- // Extract parameter type- if funcType := funcArg.Func_type(); funcType != nil {- if funcType.Typename() != nil {- signature.WriteString(extractTypeName(funcType.Typename()))- }- }- }- }- }- }-- signature.WriteString(")")- return signature.String()-}--// EnterCreatetrigstmt is called when entering a create trigger statement-func (e *metadataExtractor) EnterCreatetrigstmt(ctx *parser.CreatetrigstmtContext) {- if e.err != nil {- return- }-- // Extract trigger name using normalization to properly handle quoted identifiers- triggerNameCtx := ctx.Name()- if triggerNameCtx == nil {- return- }-- triggerName := pgparser.NormalizePostgreSQLName(triggerNameCtx)- if triggerName == "" {- return- }-- // Extract table name that the trigger is on using normalization to properly handle quoted identifiers- qualifiedNameCtx := ctx.Qualified_name()- if qualifiedNameCtx == nil {- return- }-- // Use NormalizePostgreSQLQualifiedName to handle quotes correctly- parts := pgparser.NormalizePostgreSQLQualifiedName(qualifiedNameCtx)- var schemaName, tableName string- if len(parts) == 1 {- schemaName = e.currentSchema- tableName = parts[0]- } else if len(parts) == 2 {- schemaName = parts[0]- tableName = parts[1]- }-- if tableName == "" {- return- }-- // Build trigger metadata- triggerMetadata := &storepb.TriggerMetadata{- Name: triggerName,- Body: e.buildTriggerDefinition(ctx),- }-- // Store trigger with table key- tableKey := fmt.Sprintf("%s.%s", schemaName, tableName)- if e.triggers[tableKey] == nil {- e.triggers[tableKey] = []*storepb.TriggerMetadata{}- }- e.triggers[tableKey] = append(e.triggers[tableKey], triggerMetadata)-}--// buildTriggerDefinition builds the trigger definition from the CREATE TRIGGER context-func (e *metadataExtractor) buildTriggerDefinition(ctx *parser.CreatetrigstmtContext) string {- // Build trigger definition by extracting individual components- // This is more reliable than trying to fix the concatenated text-- var parts []string-- // CREATE TRIGGER (uppercase to match system catalog)- parts = append(parts, "CREATE TRIGGER")-- // Trigger name- if nameCtx := ctx.Name(); nameCtx != nil {- triggerName := strings.ToLower(nameCtx.GetText())- parts = append(parts, triggerName)- }-- // Timing (BEFORE/AFTER/INSTEAD OF) - uppercase- if actionTimeCtx := ctx.Triggeractiontime(); actionTimeCtx != nil {- timing := strings.ToUpper(actionTimeCtx.GetText())- // Handle "INSTEADOF" -> "INSTEAD OF"- if timing == "INSTEADOF" {- timing = "INSTEAD OF"- }- parts = append(parts, timing)- }-- // Events (INSERT/UPDATE/DELETE) - uppercase with OR separators- if eventsCtx := ctx.Triggerevents(); eventsCtx != nil {- events := strings.ToUpper(eventsCtx.GetText())- // Parse and format events with proper OR separators- events = e.formatTriggerEvents(events)- parts = append(parts, events)- }-- // ON table_name with schema qualification- parts = append(parts, "ON")- if qualifiedNameCtx := ctx.Qualified_name(); qualifiedNameCtx != nil {- tableName := strings.ToLower(qualifiedNameCtx.GetText())- // Add schema qualification if not present (sync always includes schema)- if !strings.Contains(tableName, ".") {- schemaName := "public"- if e.currentSchema != "" {- schemaName = e.currentSchema- }- tableName = schemaName + "." + tableName- }- parts = append(parts, tableName)- }-- // FOR EACH ROW/STATEMENT - uppercase- if forSpecCtx := ctx.Triggerforspec(); forSpecCtx != nil {- forSpec := strings.ToUpper(forSpecCtx.GetText())- // Handle "FOREACHROW" -> "FOR EACH ROW", etc.- forSpec = e.formatTriggerForSpec(forSpec)- parts = append(parts, forSpec)- }-- // WHEN condition (optional) - uppercase- if whenCtx := ctx.Triggerwhen(); whenCtx != nil {- whenClause := strings.ToUpper(whenCtx.GetText())- parts = append(parts, "WHEN", whenClause)- }-- // EXECUTE FUNCTION/PROCEDURE - uppercase- parts = append(parts, "EXECUTE")-- if funcOrProcCtx := ctx.Function_or_procedure(); funcOrProcCtx != nil {- funcType := strings.ToUpper(funcOrProcCtx.GetText())- parts = append(parts, funcType)- } else {- // Default to FUNCTION if not specified- parts = append(parts, "FUNCTION")- }-- // Function name with schema qualification- if funcNameCtx := ctx.Func_name(); funcNameCtx != nil {- funcName := strings.ToLower(funcNameCtx.GetText())-- // Add schema qualification if not present (sync always includes schema)- if !strings.Contains(funcName, ".") {- schemaName := "public"- if e.currentSchema != "" {- schemaName = e.currentSchema- }- funcName = schemaName + "." + funcName- }-- // Function arguments- funcCall := funcName + "("- if funcArgsCtx := ctx.Triggerfuncargs(); funcArgsCtx != nil {- args := strings.ToLower(funcArgsCtx.GetText())- funcCall += args- }- funcCall += ")"-- parts = append(parts, funcCall)- }-- return strings.Join(parts, " ")-}--// formatTriggerEvents formats trigger events with proper OR separators, preserving order from DDL-func (*metadataExtractor) formatTriggerEvents(events string) string {- // The system catalog returns events in lowercase, and the expected format seems to preserve- // the order as written in the DDL. Let's normalize to match the expected lowercase format-- // Handle common concatenated patterns first- eventsLower := strings.ToLower(events)-- // Direct mappings for common patterns to match expected system catalog format- eventMap := map[string]string{- "insertorupdateordelete": "insert or delete or update", // Expected format from test- "insertordelete": "insert or delete",- "insertorupdate": "insert or update",- "updateordelete": "delete or update",- "deleteorupdate": "delete or update",- "updateorinsert": "insert or update",- "deleteorinsert": "insert or delete",- "update": "update",- "delete": "delete",- "insert": "insert",- }-- if formatted, exists := eventMap[eventsLower]; exists {- return strings.ToUpper(formatted)- }-- // For compound events, try to split and normalize while preserving original order- if strings.Contains(strings.ToUpper(events), "OR") {- parts := strings.Split(strings.ToUpper(events), "OR")- for i := range parts {- parts[i] = strings.TrimSpace(parts[i])- }- return strings.Join(parts, " OR ")- }-- // Return uppercase if no special handling needed- return strings.ToUpper(events)-}--// formatTriggerForSpec formats trigger FOR EACH specifications-func (*metadataExtractor) formatTriggerForSpec(forSpec string) string {- specMap := map[string]string{- "foreachrow": "FOR EACH ROW",- "foreachstatement": "FOR EACH STATEMENT",- "for each row": "FOR EACH ROW",- "for each statement": "FOR EACH STATEMENT",- }-- specLower := strings.ToLower(forSpec)- if formatted, exists := specMap[specLower]; exists {- return formatted- }-- // Return uppercase if not found- return strings.ToUpper(forSpec)-}--// EnterCreateextensionstmt is called when entering a create extension statement-func (e *metadataExtractor) EnterCreateextensionstmt(ctx *parser.CreateextensionstmtContext) {- if e.err != nil {- return- }-- if ctx.Name() == nil {- return- }-- extension := &storepb.ExtensionMetadata{- Name: pgparser.NormalizePostgreSQLName(ctx.Name()),- Schema: "public", // Default schema- }-- extractExtensionOptions(extension, ctx.Create_extension_opt_list())- e.extensions[extension.Name] = extension-}--// extractExtensionOptions populates Schema and Version from CREATE EXTENSION options.-func extractExtensionOptions(extension *storepb.ExtensionMetadata, optList parser.ICreate_extension_opt_listContext) {- if optList == nil {- return- }- for _, optItem := range optList.AllCreate_extension_opt_item() {- if optItem == nil {- continue- }- if optItem.SCHEMA() != nil && optItem.Name() != nil {- extension.Schema = pgparser.NormalizePostgreSQLName(optItem.Name())- }- if optItem.VERSION_P() != nil && optItem.Nonreservedword_or_sconst() != nil {- extension.Version = stripSingleQuotes(optItem.Nonreservedword_or_sconst().GetText())- }- }-}--// stripSingleQuotes removes surrounding single quotes from a SQL string constant.-func stripSingleQuotes(s string) string {- if len(s) >= 2 && s[0] == '\'' && s[len(s)-1] == '\'' {- return s[1 : len(s)-1]- }- return s-}--// EnterDefinestmt is called when entering a define statement (CREATE TYPE AS ENUM)-func (e *metadataExtractor) EnterDefinestmt(ctx *parser.DefinestmtContext) {- if e.err != nil {- return- }-- // Check if this is CREATE TYPE AS ENUM- if ctx.CREATE() != nil && ctx.TYPE_P() != nil && ctx.AS() != nil && ctx.ENUM_P() != nil {- // Extract type name- typeNames := ctx.AllAny_name()- if len(typeNames) == 0 {- return- }-- // Get the schema and enum name from the first Any_name (which should be the type name)- typeName := typeNames[0]- schemaName, enumName := e.extractSchemaAndEnumName(typeName)- schemaMetadata := e.getOrCreateSchema(schemaName)-- // Create enum metadata- enumType := &storepb.EnumTypeMetadata{- Name: enumName,- Values: []string{},- }-- // Extract enum values- if optEnumValList := ctx.Opt_enum_val_list(); optEnumValList != nil {- if enumValList := optEnumValList.Enum_val_list(); enumValList != nil {- for _, sconst := range enumValList.AllSconst() {- if sconst != nil {- value := extractStringConstant(sconst)- if value != "" {- enumType.Values = append(enumType.Values, value)- }- }- }- }- }-- if schemaMetadata.EnumTypes == nil {- schemaMetadata.EnumTypes = []*storepb.EnumTypeMetadata{}- }- schemaMetadata.EnumTypes = append(schemaMetadata.EnumTypes, enumType)- }-}--// EnterIndexstmt is called when entering a create index statement-func (e *metadataExtractor) EnterIndexstmt(ctx *parser.IndexstmtContext) {- if e.err != nil {- return- }-- // Check if this is CREATE INDEX- if ctx.CREATE() == nil || ctx.INDEX() == nil || ctx.ON() == nil {- return- }-- // Extract index name- var indexName string- if name := ctx.Name(); name != nil {- indexName = pgparser.NormalizePostgreSQLName(name)- }-- // If no explicit name, PostgreSQL will generate one - we can't predict it here- if indexName == "" {- return- }-- // Extract table/view name from relation_expr- if relationExpr := ctx.Relation_expr(); relationExpr != nil {- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {- schemaName, relationName := e.extractSchemaAndTableName(qualifiedName)-- // Try to find materialized view first, then table- var indexTarget any- var targetIndexes *[]*storepb.IndexMetadata-- if materializedView := e.findMaterializedView(schemaName, relationName); materializedView != nil {- // Index is on a materialized view- indexTarget = materializedView- if materializedView.Indexes == nil {- materializedView.Indexes = []*storepb.IndexMetadata{}- }- targetIndexes = &materializedView.Indexes- } else {- // Index is on a table - get or create the table- tableMetadata := e.getOrCreateTable(schemaName, relationName)- indexTarget = tableMetadata- targetIndexes = &tableMetadata.Indexes- }-- // If we couldn't find either table or materialized view, skip- if indexTarget == nil || targetIndexes == nil {- return- }-- // Create index metadata- index := &storepb.IndexMetadata{- Name: indexName,- Expressions: []string{},- Descending: []bool{},- Unique: false,- Primary: false,- IsConstraint: false,- Visible: false, // Match PostgreSQL database behavior- // Don't set KeyLength - PostgreSQL database doesn't return this information- }-- // Check if it's a unique index- if optUnique := ctx.Opt_unique(); optUnique != nil {- index.Unique = true- }-- // Extract index method (BTREE, HASH, GIN, GIST, etc.)- // Default to btree if not specified- index.Type = "btree"- if accessMethod := ctx.Access_method_clause(); accessMethod != nil {- if accessMethod.USING() != nil && accessMethod.Name() != nil {- index.Type = strings.ToLower(pgparser.NormalizePostgreSQLName(accessMethod.Name()))- }- }-- // Extract index parameters (columns/expressions)- if indexParams := ctx.Index_params(); indexParams != nil {- for _, indexElem := range indexParams.AllIndex_elem() {- if indexElem == nil {- continue- }-- // Extract column name or expression- var expression string- if colid := indexElem.Colid(); colid != nil {- // Simple column reference- expression = pgparser.NormalizePostgreSQLColid(colid)- } else if funcExpr := indexElem.Func_expr_windowless(); funcExpr != nil {- // Function expression - parse semantically for better compatibility- expression = e.extractFunctionExpression(ctx, funcExpr)- } else if aExpr := indexElem.A_expr(); aExpr != nil {- // General expression (in parentheses)- expression = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(aExpr)- }-- if expression != "" {- index.Expressions = append(index.Expressions, expression)-- // Extract sort order (ASC/DESC) from index element options- isDescending := false- if options := indexElem.Index_elem_options(); options != nil {- if ascDesc := options.Opt_asc_desc(); ascDesc != nil {- if ascDesc.DESC() != nil {- isDescending = true- }- // ASC is default, so we don't need to check for it explicitly- }- }- index.Descending = append(index.Descending, isDescending)- // Don't set KeyLength - PostgreSQL database doesn't return this information- }- }- }-- // Generate definition for the index- index.Definition = e.generateIndexDefinition(ctx, index, schemaName, relationName)-- // Add the index to the table or materialized view- *targetIndexes = append(*targetIndexes, index)- }- }-}--// generateIndexDefinition generates the CREATE INDEX definition for an index-func (e *metadataExtractor) generateIndexDefinition(ctx *parser.IndexstmtContext, index *storepb.IndexMetadata, schemaName, tableName string) string {- var parts []string-- // Start with CREATE [UNIQUE] INDEX- if index.Unique {- parts = append(parts, "CREATE UNIQUE INDEX")- } else {- parts = append(parts, "CREATE INDEX")- }-- // Add index name- parts = append(parts, index.Name)-- // Add ON table- parts = append(parts, "ON")- if schemaName != "" && schemaName != "public" {- parts = append(parts, fmt.Sprintf("%s.%s", schemaName, tableName))- } else {- parts = append(parts, fmt.Sprintf("public.%s", tableName))- }-- // Add USING method (default is btree)- if index.Type != "" && index.Type != "btree" {- parts = append(parts, "USING", index.Type)- } else {- parts = append(parts, "USING btree")- }-- // Add column list- if len(index.Expressions) > 0 {- columnList := make([]string, len(index.Expressions))- for i, expr := range index.Expressions {- quoted := quoteIndexExpression(expr)- // Add DESC if needed- if i < len(index.Descending) && index.Descending[i] {- columnList[i] = fmt.Sprintf("%s DESC", quoted)- } else {- columnList[i] = quoted- }- }- parts = append(parts, fmt.Sprintf("(%s)", strings.Join(columnList, ", ")))- }-- // Check for INCLUDE clause (covering index)- fullStatement := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- if includeClause := e.extractIncludeClause(fullStatement); includeClause != "" {- parts = append(parts, includeClause)- }-- // Check for WHERE clause (partial index)- if ctx.Where_clause() != nil {- whereClause := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())- // Add WHERE clause as-is- if whereClause != "" {- parts = append(parts, whereClause)- }- }-- // End with semicolon- return strings.Join(parts, " ") + ";"-}--// extractIncludeClause extracts the INCLUDE clause from a CREATE INDEX statement-func (*metadataExtractor) extractIncludeClause(statement string) string {- // Convert to lowercase for case-insensitive matching- lowerStatement := strings.ToLower(statement)-- // Find the INCLUDE keyword- includeIdx := strings.Index(lowerStatement, " include ")- if includeIdx == -1 {- return ""- }-- // Find the opening parenthesis after INCLUDE- searchStart := includeIdx + 9 // length of " include "- openParenIdx := strings.Index(lowerStatement[searchStart:], "(")- if openParenIdx == -1 {- return ""- }- openParenIdx += searchStart-- // Find the matching closing parenthesis- parenCount := 1- i := openParenIdx + 1- for i < len(lowerStatement) && parenCount > 0 {- switch lowerStatement[i] {- case '(':- parenCount++- case ')':- parenCount--- default:- // Other characters don't affect parentheses counting- }- i++- }-- if parenCount > 0 {- return "" // Unmatched parentheses- }-- // Extract the INCLUDE clause from the original statement (preserving case)- includeClause := statement[includeIdx+1 : i] // +1 to skip the leading space- return strings.TrimSpace(includeClause)-}--// generateConstraintIndexDefinition generates the CREATE INDEX definition for constraint-based indexes-func (*metadataExtractor) generateConstraintIndexDefinition(index *storepb.IndexMetadata, tableName, schemaName string) string {- var parts []string-- // Start with CREATE [UNIQUE] INDEX- if index.Unique {- parts = append(parts, "CREATE UNIQUE INDEX")- } else {- parts = append(parts, "CREATE INDEX")- }-- // Add index name- parts = append(parts, index.Name)-- // Add ON table- parts = append(parts, "ON")- if schemaName != "" && schemaName != "public" {- parts = append(parts, fmt.Sprintf("%s.%s", schemaName, tableName))- } else {- parts = append(parts, fmt.Sprintf("public.%s", tableName))- }-- // Add USING method (default is btree)- if index.Type != "" && index.Type != "btree" {- parts = append(parts, "USING", index.Type)- } else {- parts = append(parts, "USING btree")- }-- // Add column list- if len(index.Expressions) > 0 {- columnList := make([]string, len(index.Expressions))- for i, expr := range index.Expressions {- quoted := quoteIndexExpression(expr)- // Add DESC if needed- if i < len(index.Descending) && index.Descending[i] {- columnList[i] = fmt.Sprintf("%s DESC", quoted)- } else {- columnList[i] = quoted- }- }- parts = append(parts, fmt.Sprintf("(%s)", strings.Join(columnList, ", ")))- }- // End with semicolon- return strings.Join(parts, " ") + ";"-}--// extractSequenceOption extracts sequence options-func (*metadataExtractor) extractSequenceOption(ctx parser.ISeqoptelemContext, sequence *storepb.SequenceMetadata) {- if ctx == nil || sequence == nil {- return- }-- switch {- case ctx.AS() != nil && ctx.Simpletypename() != nil:- // Data type - preserve the original case- sequence.DataType = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Simpletypename())- case ctx.INCREMENT() != nil && ctx.Numericonly() != nil:- // INCREMENT BY- sequence.Increment = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())- case ctx.MINVALUE() != nil && ctx.Numericonly() != nil:- // MINVALUE- sequence.MinValue = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())- case ctx.MAXVALUE() != nil && ctx.Numericonly() != nil:- // MAXVALUE- sequence.MaxValue = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())- case ctx.START() != nil && ctx.Numericonly() != nil:- // START WITH- sequence.Start = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())- case ctx.CACHE() != nil && ctx.Numericonly() != nil:- // CACHE- sequence.CacheSize = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())- case ctx.CYCLE() != nil:- // CYCLE- sequence.Cycle = true- case ctx.NO() != nil && ctx.CYCLE() != nil:- // NO CYCLE- sequence.Cycle = false- case ctx.OWNED() != nil && ctx.BY() != nil && ctx.Any_name() != nil:- // OWNED BY table.column- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())- if len(parts) == 2 {- // Format: table.column- sequence.OwnerTable = parts[0]- sequence.OwnerColumn = parts[1]- } else if len(parts) == 3 {- // Format: schema.table.column (we only care about table.column)- sequence.OwnerTable = parts[1]- sequence.OwnerColumn = parts[2]- }- default:- // Other sequence options- }-}--// createIdentitySequence creates an identity sequence for a column with GENERATED AS IDENTITY-func (e *metadataExtractor) createIdentitySequence(table *storepb.TableMetadata, column *storepb.ColumnMetadata, schemaName string) {- // Create identity sequence name following PostgreSQL conventions- // Format: {table_name}_{column_name}_seq- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)-- // Determine sequence data type and limits based on column type- // For identity columns, use positive ranges starting from 1- var dataType, minValue, maxValue string- switch strings.ToUpper(column.Type) {- case "SMALLINT", "INT2":- dataType = "smallint"- minValue = "1"- maxValue = "32767"- case "INTEGER", "INT", "INT4":- dataType = "integer"- minValue = "1"- maxValue = "2147483647"- case "BIGINT", "INT8":- dataType = "bigint"- minValue = "1"- maxValue = "9223372036854775807"- default:- // Default to bigint for unknown types- dataType = "bigint"- minValue = "1"- maxValue = "9223372036854775807"- }-- // Create the sequence metadata- sequence := &storepb.SequenceMetadata{- Name: sequenceName,- DataType: dataType,- Start: "1",- Increment: "1",- MinValue: minValue,- MaxValue: maxValue,- Cycle: false,- CacheSize: "1",- OwnerTable: table.Name,- OwnerColumn: column.Name,- }-- // Add the sequence to the schema- schema := e.getOrCreateSchema(schemaName)- if schema.Sequences == nil {- schema.Sequences = []*storepb.SequenceMetadata{}- }- schema.Sequences = append(schema.Sequences, sequence)-- // Store in the sequences map for reference- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)- e.sequences[sequenceKey] = sequence-}--// createIdentitySequenceFromAST creates an identity sequence from ALTER COLUMN ADD IDENTITY using AST-func (e *metadataExtractor) createIdentitySequenceFromAST(table *storepb.TableMetadata, column *storepb.ColumnMetadata, schemaName string, seqOptList parser.ISeqoptlistContext) {- // Default sequence name- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)-- // Determine sequence data type and limits based on column type- // For identity columns, use positive ranges starting from 1- var dataType, minValue, maxValue string- switch strings.ToUpper(column.Type) {- case "SMALLINT", "INT2":- dataType = "smallint"- minValue = "1"- maxValue = "32767"- case "INTEGER", "INT", "INT4":- dataType = "integer"- minValue = "1"- maxValue = "2147483647"- case "BIGINT", "INT8":- dataType = "bigint"- minValue = "1"- maxValue = "9223372036854775807"- default:- // Default to bigint for unknown types- dataType = "bigint"- minValue = "1"- maxValue = "9223372036854775807"- }-- // Create the sequence metadata with defaults- sequence := &storepb.SequenceMetadata{- Name: sequenceName,- DataType: dataType,- Start: "1",- Increment: "1",- MinValue: minValue,- MaxValue: maxValue,- Cycle: false,- CacheSize: "1",- OwnerTable: table.Name,- OwnerColumn: column.Name,- }-- // Override with specified options from ALTER COLUMN statement (from AST)- if seqOptList != nil {- for _, seqElem := range seqOptList.AllSeqoptelem() {- if seqElem == nil {- continue- }- // Check for SEQUENCE NAME option to override default name- if seqElem.SEQUENCE() != nil && seqElem.NAME_P() != nil {- if anyName := seqElem.Any_name(); anyName != nil {- _, seqName := e.extractSchemaAndEnumName(anyName)- if seqName != "" {- sequenceName = seqName- sequence.Name = sequenceName- }- }- }- // Extract other sequence options- e.extractSequenceOption(seqElem, sequence)- }- }-- // Add the sequence to the schema- schema := e.getOrCreateSchema(schemaName)- if schema.Sequences == nil {- schema.Sequences = []*storepb.SequenceMetadata{}- }- schema.Sequences = append(schema.Sequences, sequence)-- // Store in the sequences map for reference- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)- e.sequences[sequenceKey] = sequence-}--// extractSchemaAndEnumName extracts schema and enum name from Any_name context-func (e *metadataExtractor) extractSchemaAndEnumName(ctx parser.IAny_nameContext) (string, string) {- if ctx == nil {- return e.currentSchema, ""- }-- parts := pgparser.NormalizePostgreSQLAnyName(ctx)- if len(parts) == 1 {- return e.currentSchema, parts[0]- } else if len(parts) == 2 {- return parts[0], parts[1]- }- return e.currentSchema, ""-}--// extractStringConstant extracts string value from Sconst context-func extractStringConstant(ctx parser.ISconstContext) string {- if ctx == nil {- return ""- }- // Get the text and remove surrounding quotes- text := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- if len(text) >= 2 && text[0] == '\'' && text[len(text)-1] == '\'' {- // Remove surrounding single quotes and handle escaped quotes- result := text[1 : len(text)-1]- result = strings.ReplaceAll(result, "''", "'")- return result- }- return text-}--// normalizeDefaultValue normalizes default values to match PostgreSQL's stored format-func (*metadataExtractor) normalizeDefaultValue(rawDefault string, column *storepb.ColumnMetadata, schemaName string) string {- if rawDefault == "" {- return ""- }-- // Handle nextval() for sequences - add schema and regclass cast- if strings.Contains(rawDefault, "nextval(") {- // Pattern: nextval('sequence_name') -> nextval('schema.sequence_name'::regclass)- // Extract sequence name from nextval('sequence_name')- if start := strings.Index(rawDefault, "nextval('"); start != -1 {- start += len("nextval('")- if end := strings.Index(rawDefault[start:], "'"); end != -1 {- sequenceName := rawDefault[start : start+end]-- // If sequence name doesn't have schema prefix, add current schema- if !strings.Contains(sequenceName, ".") {- return fmt.Sprintf("nextval('%s.%s'::regclass)", schemaName, sequenceName)- }- return fmt.Sprintf("nextval('%s'::regclass)", sequenceName)- }- }- }-- // Handle ENUM default values - add type cast- if column.Type != "" && strings.Contains(column.Type, ".") {- // If column type is schema-qualified (e.g., "public.status_enum")- // and default is a string literal, add type cast- if strings.HasPrefix(rawDefault, "'") && strings.HasSuffix(rawDefault, "'") {- return fmt.Sprintf("%s::%s", rawDefault, column.Type)- }- }-- return rawDefault-}--// extractFunctionExpression extracts function expressions preserving the original user input.-func (*metadataExtractor) extractFunctionExpression(ctx *parser.IndexstmtContext, funcExpr parser.IFunc_expr_windowlessContext) string {- if funcExpr == nil {- return ""- }-- // Return the original text without any normalization- // This preserves the user's original input format- return ctx.GetParser().GetTokenStream().GetTextFromRuleContext(funcExpr)-}--// extractViewDependencies analyzes view definitions to extract dependencies using GetQuerySpan-func extractViewDependencies(schemaMetadata *storepb.DatabaseSchemaMetadata) {- // Extract dependencies for each view- for _, schema := range schemaMetadata.Schemas {- for _, view := range schema.Views {- view.DependencyColumns = getViewDependencies(view.Definition, schema.Name, schemaMetadata)- }-- for _, mv := range schema.MaterializedViews {- mv.DependencyColumns = getViewDependencies(mv.Definition, schema.Name, schemaMetadata)- }- }-}--// getViewDependencies extracts the tables/views that a view depends on using GetQuerySpan-func getViewDependencies(viewDef string, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {- // viewDef is already a SELECT statement extracted from the parsed CREATE VIEW statement- queryStatement := strings.TrimSpace(viewDef)-- // Use GetQuerySpan with the full schema metadata so it can resolve table/view references- span, err := pgparser.GetQuerySpan(- context.Background(),- base.GetQuerySpanContext{- GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {- // Return the full schema metadata so GetQuerySpan can resolve references- dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, nil, nil, storepb.Engine_POSTGRES, false)- return databaseName, dbMetadata, nil- },- ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {- // Return empty list - we don't need actual database names for dependency extraction- return []string{}, nil- },- },- base.Statement{Text: queryStatement},- "", // database- schemaName,- false, // case sensitive- )-- // If error parsing query span, return empty dependencies- if err != nil {- return []*storepb.DependencyColumn{} // nolint:nilerr- }-- // Collect unique dependencies- dependencyMap := make(map[string]*storepb.DependencyColumn)- for sourceColumn := range span.SourceColumns {- // Create dependency key in format: schema.table- key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)- if _, exists := dependencyMap[key]; !exists {- dependencyMap[key] = &storepb.DependencyColumn{- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- Column: "*", // Use wildcard since we're tracking table-level dependencies- }- }- }-- // Convert map to slice- var dependencies []*storepb.DependencyColumn- for _, dep := range dependencyMap {- dependencies = append(dependencies, dep)- }-- return dependencies-}--// TODO: Add support for more PostgreSQL constructs if needed-// (e.g., triggers, materialized views, custom types, etc.)diff --git a/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.go b/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.godeleted file mode 100644index 3ca50ec23b..0000000000--- a/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.go+++ /dev/null@@ -1,2318 +0,0 @@-package pg--import (- "context"- "fmt"- "strings"- "testing"- "time"-- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/schema"- "github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"- "github.com/bytebase/bytebase/backend/store/model"-)--// TestGetDatabaseMetadataWithTestcontainer tests the get_database_metadata function-// by comparing its output with the metadata retrieved from a real PostgreSQL instance.-//-//nolint:tparallel-func TestGetDatabaseMetadataWithTestcontainer(t *testing.T) {- ctx := context.Background()-- // Start shared PostgreSQL container for all subtests- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- // Test cases with various PostgreSQL features- testCases := []struct {- name string- ddl string- }{- {- name: "bytebase_schema",- ddl: `- CREATE TABLE employee (- emp_no SERIAL NOT NULL,- birth_date DATE NOT NULL,- first_name TEXT NOT NULL,- last_name TEXT NOT NULL,- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,- hire_date DATE NOT NULL,- PRIMARY KEY (emp_no)-);--CREATE INDEX idx_employee_hire_date ON employee (hire_date);--CREATE TABLE department (- dept_no TEXT NOT NULL,- dept_name TEXT NOT NULL,- PRIMARY KEY (dept_no),- UNIQUE (dept_name)-);--CREATE TABLE dept_manager (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE dept_emp (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE title (- emp_no INT NOT NULL,- title TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, title, from_date)-);--CREATE TABLE salary (- emp_no INT NOT NULL,- amount INT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, from_date)-);--CREATE INDEX idx_salary_amount ON salary (amount);--CREATE TABLE audit (- id SERIAL PRIMARY KEY,- operation TEXT NOT NULL,- query TEXT,- user_name TEXT NOT NULL,- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_audit_operation ON audit (operation);-CREATE INDEX idx_audit_username ON audit (user_name);-CREATE INDEX idx_audit_changed_at ON audit (changed_at);--CREATE OR REPLACE FUNCTION log_dml_operations() RETURNS TRIGGER AS $$-BEGIN- IF (TG_OP = 'INSERT') THEN- INSERT INTO audit (operation, query, user_name)- VALUES ('INSERT', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'UPDATE') THEN- INSERT INTO audit (operation, query, user_name)- VALUES ('UPDATE', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'DELETE') THEN- INSERT INTO audit (operation, query, user_name)- VALUES ('DELETE', current_query(), current_user);- RETURN OLD;- END IF;- RETURN NULL;-END;-$$ LANGUAGE plpgsql;---- only log update and delete, otherwise, it will cause too much change.-CREATE TRIGGER salary_log_trigger-AFTER UPDATE OR DELETE ON salary-FOR EACH ROW-EXECUTE FUNCTION log_dml_operations();--CREATE OR REPLACE VIEW dept_emp_latest_date AS-SELECT- emp_no,- MAX(- from_date) AS from_date,- MAX(- to_date) AS to_date-FROM- dept_emp-GROUP BY- emp_no;---- shows only the current department for each employee-CREATE OR REPLACE VIEW current_dept_emp AS-SELECT- l.emp_no,- dept_no,- l.from_date,- l.to_date-FROM- dept_emp d- INNER JOIN dept_emp_latest_date l ON d.emp_no = l.emp_no- AND d.from_date = l.from_date- AND l.to_date = d.to_date;- `,- },- {- name: "basic_tables_with_constraints",- ddl: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- updated_at TIMESTAMP-);--CREATE TABLE posts (- id SERIAL PRIMARY KEY,- user_id INTEGER NOT NULL,- title VARCHAR(200) NOT NULL,- content TEXT,- published BOOLEAN DEFAULT false,- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE-);--CREATE INDEX idx_posts_user_id ON posts(user_id);-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;-`,- },- {- name: "sequences_and_custom_types",- ddl: `-CREATE TYPE status_enum AS ENUM ('pending', 'active', 'inactive', 'deleted');-CREATE TYPE mood AS ENUM ('happy', 'sad', 'neutral');--CREATE SEQUENCE custom_id_seq START WITH 1000 INCREMENT BY 10;--CREATE TABLE items (- id INTEGER DEFAULT nextval('custom_id_seq') PRIMARY KEY,- name VARCHAR(100) NOT NULL,- status status_enum DEFAULT 'pending',- user_mood mood-);-`,- },- {- name: "views_and_functions",- ddl: `-CREATE TABLE employees (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- department VARCHAR(50),- salary DECIMAL(10, 2)-);--CREATE VIEW active_employees AS-SELECT id,- name,- department- FROM employees-WHERE (department IS NOT NULL);--CREATE FUNCTION get_employee_count(dept VARCHAR) RETURNS INTEGER AS $$-BEGIN- RETURN (SELECT COUNT(*) FROM employees WHERE department = dept);-END;-$$ LANGUAGE plpgsql;--CREATE FUNCTION calculate_bonus(emp_id INTEGER) RETURNS DECIMAL AS $$-DECLARE- emp_salary DECIMAL;-BEGIN- SELECT salary INTO emp_salary FROM employees WHERE id = emp_id;- RETURN emp_salary * 0.1;-END;-$$ LANGUAGE plpgsql;-`,- },- {- name: "partitioned_tables",- ddl: `-CREATE TABLE sales (- id SERIAL,- sale_date DATE NOT NULL,- amount DECIMAL(10, 2) NOT NULL,- region VARCHAR(50) NOT NULL-) PARTITION BY RANGE (sale_date);--CREATE TABLE sales_2023_q1 PARTITION OF sales-FOR VALUES FROM ('2023-01-01') TO ('2023-04-01');--CREATE TABLE sales_2023_q2 PARTITION OF sales-FOR VALUES FROM ('2023-04-01') TO ('2023-07-01');--CREATE TABLE sales_2023_q3 PARTITION OF sales-FOR VALUES FROM ('2023-07-01') TO ('2023-10-01');--CREATE TABLE sales_2023_q4 PARTITION OF sales-FOR VALUES FROM ('2023-10-01') TO ('2024-01-01');-`,- },- {- name: "extensions_and_advanced_features",- ddl: `-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";--CREATE TABLE documents (- id UUID DEFAULT uuid_generate_v4() PRIMARY KEY,- title VARCHAR(200) NOT NULL,- content TEXT,- tags TEXT[],- metadata JSONB,- created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_documents_metadata ON documents USING GIN (metadata);-CREATE INDEX idx_documents_tags ON documents USING GIN (tags);-`,- },- {- name: "indexes_with_asc_desc",- ddl: `-CREATE TABLE orders (- id SERIAL PRIMARY KEY,- customer_id INTEGER NOT NULL,- order_date DATE NOT NULL,- total_amount DECIMAL(10, 2) NOT NULL,- status VARCHAR(20)-);---- Index with explicit DESC-CREATE INDEX idx_orders_date_desc ON orders(order_date DESC);---- Index with multiple columns, mixed ASC/DESC-CREATE INDEX idx_orders_customer_date ON orders(customer_id ASC, order_date DESC);---- Index with expressions and DESC-CREATE INDEX idx_orders_year_month ON orders(EXTRACT( year FROM order_date ) DESC, EXTRACT( month FROM order_date ) ASC);---- Unique index with DESC-CREATE UNIQUE INDEX idx_orders_customer_status ON orders(customer_id, status DESC) WHERE status IS NOT NULL;-`,- },- {- name: "materialized_views_and_triggers",- ddl: `-CREATE TABLE audit_log (- id SERIAL PRIMARY KEY,- table_name VARCHAR(50) NOT NULL,- operation VARCHAR(10) NOT NULL,- user_id INTEGER,- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- old_values JSONB,- new_values JSONB-);--CREATE TABLE users_mv (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL,- email VARCHAR(100) NOT NULL,- last_login TIMESTAMP,- login_count INTEGER DEFAULT 0-);---- Materialized view for user statistics-CREATE MATERIALIZED VIEW user_stats AS-SELECT count(*) AS total_users,- count(- CASE- WHEN (last_login > (CURRENT_DATE - '30 days'::interval)) THEN 1- ELSE NULL::integer- END) AS active_users,- avg(login_count) AS avg_login_count-FROM public.users_mv-WITH DATA;---- Index on materialized view-CREATE INDEX idx_user_stats_total ON user_stats(total_users);---- Trigger function for audit logging-CREATE OR REPLACE FUNCTION audit_trigger_function()-RETURNS TRIGGER AS $$-BEGIN- INSERT INTO audit_log (table_name, operation, user_id, old_values, new_values)- VALUES (- TG_TABLE_NAME,- TG_OP,- COALESCE(NEW.id, OLD.id),- CASE WHEN TG_OP = 'DELETE' THEN row_to_json(OLD) ELSE NULL END,- CASE WHEN TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN row_to_json(NEW) ELSE NULL END- );- RETURN COALESCE(NEW, OLD);-END;-$$ LANGUAGE plpgsql;---- Trigger on users table-CREATE TRIGGER users_audit_trigger- AFTER INSERT OR UPDATE OR DELETE ON users_mv- FOR EACH ROW- EXECUTE FUNCTION audit_trigger_function();---- Stored procedure with parameters-CREATE OR REPLACE PROCEDURE refresh_user_stats()-LANGUAGE plpgsql AS $$-BEGIN- REFRESH MATERIALIZED VIEW user_stats;- INSERT INTO audit_log (table_name, operation, timestamp)- VALUES ('user_stats', 'REFRESH', CURRENT_TIMESTAMP);-END;-$$;-`,- },- {- name: "cross_schema_references",- ddl: `--- Create additional schemas-CREATE SCHEMA hr;-CREATE SCHEMA finance;---- Tables in hr schema-CREATE TABLE hr.departments (- id SERIAL PRIMARY KEY,- name VARCHAR(50) NOT NULL UNIQUE,- manager_id INTEGER-);--CREATE TABLE hr.employees (- id SERIAL PRIMARY KEY,- first_name VARCHAR(50) NOT NULL,- last_name VARCHAR(50) NOT NULL,- department_id INTEGER NOT NULL,- salary DECIMAL(10, 2),- hire_date DATE DEFAULT CURRENT_DATE,- CONSTRAINT fk_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id)-);---- Self-referencing foreign key for manager-ALTER TABLE hr.departments-ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES hr.employees(id);---- Tables in finance schema-CREATE TABLE finance.budgets (- id SERIAL PRIMARY KEY,- department_id INTEGER NOT NULL,- fiscal_year INTEGER NOT NULL,- allocated_amount DECIMAL(12, 2) NOT NULL,- spent_amount DECIMAL(12, 2) DEFAULT 0.00,- -- Cross-schema foreign key- CONSTRAINT fk_budget_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id),- CONSTRAINT unique_budget_year UNIQUE (department_id, fiscal_year)-);---- View that joins across schemas-CREATE VIEW finance.department_spending AS-SELECT d.name AS department_name,- b.fiscal_year,- b.allocated_amount,- b.spent_amount,- (b.allocated_amount - b.spent_amount) AS remaining_budget-FROM (hr.departments d- JOIN finance.budgets b ON ((d.id = b.department_id)));-`,- },- {- name: "advanced_indexes_and_constraints",- ddl: `-CREATE EXTENSION IF NOT EXISTS btree_gin;--CREATE TABLE products (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- description TEXT,- price DECIMAL(10, 2),- category_tags TEXT[],- metadata JSONB,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- sku VARCHAR(50),- coordinates POINT-);---- BRIN index for time-series data-CREATE INDEX idx_products_created_brin ON products USING BRIN (created_at);---- Hash index-CREATE INDEX idx_products_sku_hash ON products USING HASH (sku);---- Covering index (INCLUDE clause)-CREATE INDEX idx_products_category_include ON products (price) INCLUDE (name, description);---- Partial index with complex condition-CREATE INDEX idx_expensive_products ON products (price, category_tags)-WHERE price > 100.00 AND array_length(category_tags, 1) > 0;---- Expression index with functions-CREATE INDEX idx_products_name_lower ON products (lower(name::text));-CREATE INDEX idx_products_price_rounded ON products (round(price));---- Multi-column GIN index-CREATE INDEX idx_products_tags_meta ON products USING GIN (category_tags, metadata);---- Simple check constraints (avoiding complex exclusion constraints)-CREATE TABLE orders_advanced (- id SERIAL PRIMARY KEY,- product_id INTEGER NOT NULL,- quantity INTEGER NOT NULL,- order_date DATE DEFAULT CURRENT_DATE,- CONSTRAINT fk_product FOREIGN KEY (product_id) REFERENCES products(id),- CONSTRAINT valid_quantity CHECK (quantity > 0 AND quantity <= 1000),- CONSTRAINT recent_order CHECK (order_date >= CURRENT_DATE - INTERVAL '1 year')-);-`,- },- {- name: "geometric_and_network_types",- ddl: `--- Using built-in geometric and network types (no PostGIS required)-CREATE TABLE locations (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- coordinates POINT,- boundary_box BOX,- service_area POLYGON,- delivery_path PATH,- center_point CIRCLE,- route_line LSEG-);--CREATE TABLE network_devices (- id SERIAL PRIMARY KEY,- hostname VARCHAR(100) NOT NULL,- ip_address INET,- subnet CIDR,- mac_address MACADDR,- ipv6_address INET,- device_config JSONB-);---- Geometric indexes (using GIST for geometric types)-CREATE INDEX idx_locations_coordinates ON locations USING GIST (coordinates);-CREATE INDEX idx_locations_service_area ON locations USING GIST (service_area);---- Network type indexes-CREATE INDEX idx_devices_ip ON network_devices (ip_address);--- Skip GIST index on CIDR as it doesn't have a default operator class-CREATE INDEX idx_devices_subnet ON network_devices (subnet);---- Range types-CREATE TABLE reservations (- id SERIAL PRIMARY KEY,- resource_id INTEGER NOT NULL,- date_range DATERANGE NOT NULL,- time_range TSRANGE,- price_range NUMRANGE,- capacity_range INT4RANGE-);---- Index on range types-CREATE INDEX idx_reservations_date_range ON reservations USING GIST (date_range);---- Full-text search types-CREATE TABLE documents_fts (- id SERIAL PRIMARY KEY,- title VARCHAR(200) NOT NULL,- content TEXT,- search_vector TSVECTOR,- keywords TSQUERY-);---- Full-text search index-CREATE INDEX idx_documents_search ON documents_fts USING GIN (search_vector);---- Update trigger for full-text search-CREATE OR REPLACE FUNCTION update_search_vector() RETURNS TRIGGER AS $$-BEGIN- NEW.search_vector := to_tsvector('english', COALESCE(NEW.title, '') || ' ' || COALESCE(NEW.content, ''));- RETURN NEW;-END;-$$ LANGUAGE plpgsql;--CREATE TRIGGER trigger_update_search- BEFORE INSERT OR UPDATE ON documents_fts- FOR EACH ROW- EXECUTE FUNCTION update_search_vector();-`,- },- {- name: "routing_rules_pattern",- ddl: `--- Test the routing rules pattern (view with INSERT/UPDATE/DELETE rules)--- This is the exact pattern from the user's scenario---- Create a regular table (not partitioned to avoid test issues)-CREATE TABLE old_name (- id BIGINT NOT NULL PRIMARY KEY,- created_at TIMESTAMP WITH TIME ZONE NOT NULL,- updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,- name VARCHAR(100),- data JSONB,- status VARCHAR(20) DEFAULT 'active'::character varying-);---- Create indexes-CREATE INDEX idx_old_name_status ON old_name (status);-CREATE INDEX idx_old_name_created ON old_name (created_at);---- Create the view that will act as the new interface-CREATE VIEW new_name AS-SELECT- id,- created_at,- updated_at,- name,- data,- status-FROM old_name;---- Create routing rules to redirect operations from view to table-CREATE RULE new_name_delete AS- ON DELETE TO new_name- DO INSTEAD DELETE FROM old_name- WHERE old_name.id = OLD.id;--CREATE RULE new_name_insert AS- ON INSERT TO new_name- DO INSTEAD INSERT INTO old_name (id, created_at, updated_at, name, data, status)- VALUES (NEW.id, NEW.created_at, NEW.updated_at, NEW.name, NEW.data, NEW.status);--CREATE RULE new_name_update AS- ON UPDATE TO new_name- DO INSTEAD UPDATE old_name- SET- created_at = NEW.created_at,- updated_at = NEW.updated_at,- name = NEW.name,- data = NEW.data,- status = NEW.status- WHERE old_name.id = OLD.id;----- Create another table with rules (not on a view)-CREATE TABLE audit_table (- id SERIAL PRIMARY KEY,- table_name VARCHAR(50),- operation VARCHAR(10),- user_name VARCHAR(50),- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE TABLE monitored_table (- id SERIAL PRIMARY KEY,- data TEXT-);---- Create a rule on a regular table (not a view)-CREATE RULE log_monitored_inserts AS- ON INSERT TO monitored_table- DO ALSO INSERT INTO audit_table (table_name, operation, user_name)- VALUES ('monitored_table', 'INSERT', current_user);--CREATE RULE log_monitored_deletes AS- ON DELETE TO monitored_table- DO ALSO INSERT INTO audit_table (table_name, operation, user_name)- VALUES ('monitored_table', 'DELETE', current_user);-`,- },- {- name: "table_inheritance_and_partitioning",- ddl: `--- Skip table inheritance as it's not supported--- Just test partitioning features---- List partitioning (without unique constraints that would cause issues)-CREATE TABLE events (- id BIGINT,- event_type VARCHAR(20) NOT NULL,- event_data JSONB,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-) PARTITION BY LIST (event_type);--CREATE TABLE events_user PARTITION OF events-FOR VALUES IN ('user_login', 'user_logout', 'user_register');--CREATE TABLE events_system PARTITION OF events-FOR VALUES IN ('system_start', 'system_stop', 'system_error');--CREATE TABLE events_audit PARTITION OF events-FOR VALUES IN ('data_change', 'permission_change', 'config_change');---- Hash partitioning (without primary key to avoid partitioning column requirement)-CREATE TABLE user_sessions (- session_id UUID DEFAULT gen_random_uuid(),- user_id INTEGER NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- last_activity TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- ip_address INET-) PARTITION BY HASH (user_id);--CREATE TABLE user_sessions_0 PARTITION OF user_sessions-FOR VALUES WITH (modulus 4, remainder 0);--CREATE TABLE user_sessions_1 PARTITION OF user_sessions-FOR VALUES WITH (modulus 4, remainder 1);--CREATE TABLE user_sessions_2 PARTITION OF user_sessions-FOR VALUES WITH (modulus 4, remainder 2);--CREATE TABLE user_sessions_3 PARTITION OF user_sessions-FOR VALUES WITH (modulus 4, remainder 3);---- Unlogged table for temporary data-CREATE UNLOGGED TABLE temp_calculations (- id SERIAL PRIMARY KEY,- calculation_data JSONB,- result DECIMAL(15, 6),- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);-`,- },- {- name: "array_types_comprehensive",- ddl: `--- Create custom enum for testing custom type arrays-CREATE TYPE status_enum AS ENUM ('active', 'inactive', 'pending');-CREATE TYPE priority_level AS ENUM ('low', 'medium', 'high', 'urgent');---- Table with comprehensive array type coverage-CREATE TABLE array_types_test (- id SERIAL PRIMARY KEY,-- -- Integer array types (different aliases)- int_array int[],- int4_array int4[],- integer_array integer[],- bigint_array bigint[],- int8_array int8[],- smallint_array smallint[],- int2_array int2[],-- -- Multi-dimensional arrays (should normalize to same as single dimension)- int_multi_array int[][],- text_multi_array text[][][],-- -- Floating point array types- real_array real[],- float4_array float4[],- double_precision_array double precision[],- float8_array float8[],-- -- Character array types with and without precision- char_array char[],- char_with_length char(10)[],- varchar_array varchar[],- varchar_with_length varchar(255)[],- character_array character[],- character_with_length character(50)[],- character_varying_array character varying[],- character_varying_with_length character varying(100)[],- text_array text[],-- -- Boolean arrays- bool_array bool[],- boolean_array boolean[],-- -- Numeric arrays with precision- numeric_array numeric[],- numeric_with_precision numeric(10,2)[],- decimal_array decimal[],- decimal_with_precision decimal(15,4)[],- money_array money[],-- -- Date/time arrays- date_array date[],- time_array time[],- time_with_tz time with time zone[],- timetz_array timetz[],- timestamp_array timestamp[],- timestamp_with_tz timestamp with time zone[],- timestamptz_array timestamptz[],- interval_array interval[],-- -- Binary and UUID arrays- bytea_array bytea[],- uuid_array uuid[],-- -- JSON arrays- json_array json[],- jsonb_array jsonb[],-- -- Network address arrays- inet_array inet[],- cidr_array cidr[],- macaddr_array macaddr[],- macaddr8_array macaddr8[],-- -- Geometric type arrays- point_array point[],- line_array line[],- lseg_array lseg[],- box_array box[],- path_array path[],- polygon_array polygon[],- circle_array circle[],-- -- Full-text search arrays- tsvector_array tsvector[],- tsquery_array tsquery[],-- -- Range type arrays- int4range_array int4range[],- int8range_array int8range[],- numrange_array numrange[],- tsrange_array tsrange[],- tstzrange_array tstzrange[],- daterange_array daterange[],-- -- Multi-range type arrays (PostgreSQL 14+)- int4multirange_array int4multirange[],- int8multirange_array int8multirange[],- nummultirange_array nummultirange[],- tsmultirange_array tsmultirange[],- tstzmultirange_array tstzmultirange[],- datemultirange_array datemultirange[],-- -- Bit string arrays- bit_array bit[],- bit_with_length bit(8)[],- bit_varying_array bit varying[],- varbit_array varbit[],- varbit_with_length varbit(16)[],-- -- Object identifier arrays- oid_array oid[],- regclass_array regclass[],- regproc_array regproc[],- regprocedure_array regprocedure[],- regoper_array regoper[],- regoperator_array regoperator[],- regtype_array regtype[],- regconfig_array regconfig[],- regdictionary_array regdictionary[],- regnamespace_array regnamespace[],- regrole_array regrole[],- regcollation_array regcollation[],-- -- System type arrays- tid_array tid[],- xid_array xid[],- xid8_array xid8[],- cid_array cid[],- pg_lsn_array pg_lsn[],-- -- Custom enum arrays (should use fallback logic)- status_array status_enum[],- priority_array priority_level[],-- -- XML array- xml_array xml[],-- -- Other special arrays (note: record[] not allowed in table columns)- jsonpath_array jsonpath[],-- -- Additional system types (compatible with PostgreSQL 16)- txid_snapshot_array txid_snapshot[],-- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);---- Add some sample data to verify the schema works-INSERT INTO array_types_test (- int_array,- varchar_with_length,- bool_array,- status_array,- json_array-) VALUES (- ARRAY[1, 2, 3],- ARRAY['test1', 'test2'],- ARRAY[true, false, true],- ARRAY['active'::status_enum, 'pending'::status_enum],- ARRAY['{"key": "value1"}'::json, '{"key": "value2"}'::json]-);-`,- },- }-- for _, tc := range testCases {- t.Run(tc.name, func(t *testing.T) {- t.Parallel()-- // Create a new database for each test case with unique name- dbName := fmt.Sprintf("test_%s_%d", strings.ReplaceAll(tc.name, "-", "_"), time.Now().UnixNano()%1000000)- _, err := container.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))- require.NoError(t, err)-- // Create driver for the test database- driver, err := createPgDriver(ctx, container.GetHost(), container.GetPort(), dbName)- require.NoError(t, err)- defer driver.Close(ctx)-- // Execute the DDL- _, err = driver.GetDB().ExecContext(ctx, tc.ddl)- require.NoError(t, err)-- // Get metadata using Driver.SyncDBSchema- syncMetadata, err := driver.SyncDBSchema(ctx)- require.NoError(t, err)-- // Get metadata using get_database_metadata- parseMetadata, err := GetDatabaseMetadata(tc.ddl)- require.NoError(t, err)-- // Compare the two metadata structures- compareMetadata(t, syncMetadata, parseMetadata)-- // Additional validation: use schema differ to ensure no differences detected- validateWithSchemaDiffer(t, tc.name, syncMetadata, parseMetadata)-- // Validate rules are handled correctly (no cycles, proper dumping)- validateRoutingRules(t, syncMetadata)-- // Additional validation for array types test case- if tc.name == "array_types_comprehensive" {- validateArrayTypes(t, syncMetadata, parseMetadata)- }- })- }-}--// compareMetadata compares metadata from sync.go and get_database_metadata-func compareMetadata(t *testing.T, syncMeta, parseMeta *storepb.DatabaseSchemaMetadata) {- // Compare schemas- require.Equal(t, len(syncMeta.Schemas), len(parseMeta.Schemas), "number of schemas should match")-- // Find the public schema in both- var syncPublic, parsePublic *storepb.SchemaMetadata- for _, schema := range syncMeta.Schemas {- if schema.Name == "public" {- syncPublic = schema- break- }- }- for _, schema := range parseMeta.Schemas {- if schema.Name == "public" {- parsePublic = schema- break- }- }-- require.NotNil(t, syncPublic, "sync metadata should have public schema")- require.NotNil(t, parsePublic, "parse metadata should have public schema")-- // Compare tables- compareTables(t, syncPublic.Tables, parsePublic.Tables)-- // Compare views- compareViews(t, syncPublic.Views, parsePublic.Views)-- // Compare materialized views- compareMaterializedViews(t, syncPublic.MaterializedViews, parsePublic.MaterializedViews)-- // Compare functions- compareFunctions(t, syncPublic.Functions, parsePublic.Functions)-- // Compare procedures (part of functions in metadata)- compareProcedures(t, syncPublic.Procedures, parsePublic.Procedures)-- // Compare sequences- compareSequences(t, syncPublic.Sequences, parsePublic.Sequences)-- // Compare enums- compareEnums(t, syncPublic.EnumTypes, parsePublic.EnumTypes)-- // Note: Triggers are stored at the table level, not schema level- // They will be compared as part of table comparison-- // Compare extensions- compareExtensions(t, syncMeta.Extensions, parseMeta.Extensions)-- // Compare all schemas for cross-schema tests- compareAllSchemas(t, syncMeta.Schemas, parseMeta.Schemas)-}--// normalizeSQL normalizes SQL for comparison by:-// - Converting to lowercase-// - Removing extra whitespace-// - Removing trailing semicolons-// - Removing schema qualifiers for common schemas-// - Normalizing parentheses-func normalizeSQL(sql string) string {- // Convert to lowercase- sql = strings.ToLower(sql)-- // Replace multiple spaces/newlines with single space- sql = strings.Join(strings.Fields(sql), " ")-- // Remove trailing semicolons- sql = strings.TrimSuffix(sql, ";")-- // Remove schema qualifiers for public schema- // This handles cases like "public.table_name" -> "table_name"- sql = strings.ReplaceAll(sql, "public.", "")-- // Handle PostgreSQL's tendency to wrap WHERE conditions in parentheses- // e.g., "WHERE (condition)" -> "WHERE condition"- // We need to be careful to only remove the outermost parentheses around the entire WHERE clause- whereIndex := strings.Index(sql, "where (")- if whereIndex >= 0 {- // Find the matching closing parenthesis for the WHERE clause- afterWhere := sql[whereIndex+7:] // Skip "where ("- openCount := 1- closeIndex := -1-- // Find the matching closing parenthesis- for i, ch := range afterWhere {- if ch == '(' {- openCount++- } else if ch == ')' {- openCount--- if openCount == 0 {- closeIndex = i- break- }- }- }-- // If we found the matching closing parenthesis and it's at the end or followed by end/order/group- if closeIndex >= 0 {- beforeWhere := sql[:whereIndex+6] // Include "where "- afterCloseParen := ""- if whereIndex+7+closeIndex+1 < len(sql) {- afterCloseParen = sql[whereIndex+7+closeIndex+1:]- }-- // Check if the closing paren is at the end or followed by valid SQL keywords- if afterCloseParen == "" || strings.HasPrefix(strings.TrimSpace(afterCloseParen), "order by") ||- strings.HasPrefix(strings.TrimSpace(afterCloseParen), "group by") ||- strings.HasPrefix(strings.TrimSpace(afterCloseParen), "limit") {- // Remove the parentheses- sql = beforeWhere + afterWhere[:closeIndex] + afterCloseParen- }- }- }-- // PostgreSQL function and procedure-specific normalizations- if strings.Contains(sql, "function") || strings.Contains(sql, "procedure") {- // Normalize CREATE OR REPLACE vs CREATE- sql = strings.ReplaceAll(sql, "create or replace function", "create function")-- // Normalize parameter types to consistent forms- sql = strings.ReplaceAll(sql, "character varying", "varchar")- sql = strings.ReplaceAll(sql, "returns numeric", "returns decimal")-- // Normalize dollar quoting- sql = strings.ReplaceAll(sql, "$function$", "$$")- sql = strings.ReplaceAll(sql, "$procedure$", "$$")-- // Move LANGUAGE clause to end for consistent comparison- if strings.Contains(sql, "language plpgsql") && !strings.HasSuffix(sql, "language plpgsql") {- withoutLanguage := strings.ReplaceAll(sql, " language plpgsql", "")- withoutLanguage = strings.ReplaceAll(withoutLanguage, "language plpgsql ", "")- sql = strings.TrimSpace(withoutLanguage) + " language plpgsql"- }- }-- // Final trim- sql = strings.TrimSpace(sql)-- return sql-}--// normalizeSignature normalizes function signatures for comparison-func normalizeSignature(sig string) string {- // Convert to lowercase- sig = strings.ToLower(sig)-- // Remove extra spaces- sig = strings.Join(strings.Fields(sig), " ")-- // Remove spaces around parentheses and commas- sig = strings.ReplaceAll(sig, " (", "(")- sig = strings.ReplaceAll(sig, "( ", "(")- sig = strings.ReplaceAll(sig, " )", ")")- sig = strings.ReplaceAll(sig, ") ", ")")- sig = strings.ReplaceAll(sig, " ,", ",")- sig = strings.ReplaceAll(sig, ", ", ",")-- // Remove quotes around function names- sig = strings.ReplaceAll(sig, "\"", "")-- return sig-}--func compareTables(t *testing.T, syncTables, parseTables []*storepb.TableMetadata) {- // Log the tables found for debugging- t.Logf("Sync tables: %d", len(syncTables))- for _, table := range syncTables {- t.Logf(" - %s", table.Name)- }- t.Logf("Parse tables: %d", len(parseTables))- for _, table := range parseTables {- t.Logf(" - %s", table.Name)- }-- require.Equal(t, len(syncTables), len(parseTables), "number of tables should match")-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.TableMetadata)- for _, table := range syncTables {- syncMap[table.Name] = table- }-- parseMap := make(map[string]*storepb.TableMetadata)- for _, table := range parseTables {- parseMap[table.Name] = table- }-- // Compare each table- for name, syncTable := range syncMap {- parseTable, exists := parseMap[name]- require.True(t, exists, "table %s should exist in parsed metadata", name)-- // Compare columns- compareColumns(t, name, syncTable.Columns, parseTable.Columns)-- // Compare rules- compareRules(t, name, syncTable.Rules, parseTable.Rules)-- // Compare indexes- compareIndexes(t, name, syncTable.Indexes, parseTable.Indexes)-- // Compare foreign keys- compareForeignKeys(t, name, syncTable.ForeignKeys, parseTable.ForeignKeys)-- // Compare partitions- comparePartitions(t, name, syncTable.Partitions, parseTable.Partitions)-- // Compare triggers- compareTriggers(t, name, syncTable.Triggers, parseTable.Triggers)- }-}--func compareColumns(t *testing.T, tableName string, syncCols, parseCols []*storepb.ColumnMetadata) {- require.Equal(t, len(syncCols), len(parseCols), "table %s: number of columns should match", tableName)-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.ColumnMetadata)- for _, col := range syncCols {- syncMap[col.Name] = col- }-- for _, parseCol := range parseCols {- syncCol, exists := syncMap[parseCol.Name]- require.True(t, exists, "table %s: column %s should exist in sync metadata", tableName, parseCol.Name)-- // Compare column properties- require.Equal(t, syncCol.Type, parseCol.Type, "table %s, column %s: type should match", tableName, parseCol.Name)- require.Equal(t, syncCol.Nullable, parseCol.Nullable, "table %s, column %s: nullable should match", tableName, parseCol.Name)-- // Compare default values if both exist- hasDefaultSync := syncCol.Default != ""- hasDefaultParse := parseCol.Default != ""- if hasDefaultSync && hasDefaultParse {- // Default values might be represented differently, so we just check they exist- t.Logf("table %s, column %s: default values exist in both", tableName, parseCol.Name)- }- }-}--func compareIndexes(t *testing.T, tableName string, syncIndexes, parseIndexes []*storepb.IndexMetadata) {- // Create maps for easier comparison- syncMap := make(map[string]*storepb.IndexMetadata)- for _, idx := range syncIndexes {- syncMap[idx.Name] = idx- }-- parseMap := make(map[string]*storepb.IndexMetadata)- for _, idx := range parseIndexes {- parseMap[idx.Name] = idx- }-- // Check that both directions have the same indexes- require.Equal(t, len(syncIndexes), len(parseIndexes), "mismatch in number of indexes for table %s", tableName)-- // Compare indexes that exist in parse- for name, parseIdx := range parseMap {- syncIdx, exists := syncMap[name]- if !exists {- // Some indexes might be system-generated and not in DDL- t.Logf("table %s: index %s exists in parse but not in sync (might be implicit)", tableName, name)- continue- }-- // 1. Name - explicitly validate name consistency- require.Equal(t, syncIdx.Name, parseIdx.Name, "table %s, index %s: name should match", tableName, name)-- // 2. Primary - validate primary key flag- require.Equal(t, syncIdx.Primary, parseIdx.Primary, "table %s, index %s: primary should match", tableName, name)-- // 3. Unique - validate unique constraint flag- require.Equal(t, syncIdx.Unique, parseIdx.Unique, "table %s, index %s: unique should match", tableName, name)-- // 4. Type - validate index type- if syncIdx.Type != "" || parseIdx.Type != "" {- require.Equal(t, syncIdx.Type, parseIdx.Type, "table %s, index %s: type should match", tableName, name)- }-- // 5. Expressions - compare expressions using AST-based semantic comparison- if len(syncIdx.Expressions) == len(parseIdx.Expressions) {- for i := range syncIdx.Expressions {- // Use AST-based semantic comparison instead of string normalization- equal := ast.CompareExpressionsSemantically(syncIdx.Expressions[i], parseIdx.Expressions[i])- require.True(t, equal, "table %s, index %s: expression[%d] should be semantically equivalent. sync: %q, parse: %q",- tableName, name, i, syncIdx.Expressions[i], parseIdx.Expressions[i])- }- } else {- require.Equal(t, len(syncIdx.Expressions), len(parseIdx.Expressions), "table %s, index %s: expressions count should match", tableName, name)- }-- // 6. Descending - compare descending order for each expression- // Note: sync.go currently doesn't populate the Descending field, so we need to handle both cases- if len(syncIdx.Descending) > 0 && len(parseIdx.Descending) > 0 {- // Both have descending info, compare them- require.Equal(t, len(syncIdx.Descending), len(parseIdx.Descending), "table %s, index %s: descending array length should match", tableName, name)- for i := range syncIdx.Descending {- require.Equal(t, syncIdx.Descending[i], parseIdx.Descending[i], "table %s, index %s: descending[%d] should match", tableName, name, i)- }- } else if len(parseIdx.Descending) > 0 {- // Only parser has descending info, verify it matches the number of expressions- require.Equal(t, len(parseIdx.Expressions), len(parseIdx.Descending), "table %s, index %s: descending array should match expressions count", tableName, name)- }-- // 7. KeyLength - validate index key lengths (PostgreSQL specific - prefix lengths on expressions)- if len(syncIdx.KeyLength) > 0 || len(parseIdx.KeyLength) > 0 {- require.Equal(t, len(syncIdx.KeyLength), len(parseIdx.KeyLength), "table %s, index %s: key length array length should match", tableName, name)- for i := range syncIdx.KeyLength {- if i < len(parseIdx.KeyLength) {- require.Equal(t, syncIdx.KeyLength[i], parseIdx.KeyLength[i], "table %s, index %s: key length[%d] should match", tableName, name, i)- }- }- }-- // 8. Visible - validate index visibility (not commonly used in PostgreSQL, but validate if present)- require.Equal(t, syncIdx.Visible, parseIdx.Visible, "table %s, index %s: visible should match", tableName, name)-- // 9. Comment - validate index comment- if syncIdx.Comment != "" || parseIdx.Comment != "" {- require.Equal(t, syncIdx.Comment, parseIdx.Comment, "table %s, index %s: comment should match", tableName, name)- }-- // 10. IsConstraint - validate if index represents a constraint- require.Equal(t, syncIdx.IsConstraint, parseIdx.IsConstraint, "table %s, index %s: IsConstraint should match", tableName, name)-- // 11. Definition - validate full index definition using AST-based semantic comparison- if syncIdx.Definition != "" || parseIdx.Definition != "" {- if syncIdx.Definition != "" && parseIdx.Definition != "" {- definitionsEqual := compareIndexDefinitionsSemanticaly(syncIdx.Definition, parseIdx.Definition)- require.True(t, definitionsEqual, "table %s, index %s: definition should match\nSync: %s\nParse: %s", tableName, name, syncIdx.Definition, parseIdx.Definition)- }- }-- t.Logf("✓ Validated all IndexMetadata fields for index %s: name=%s, primary=%v, unique=%v, type=%s, expressions=%v, visible=%v, comment=%s",- name, parseIdx.Name, parseIdx.Primary, parseIdx.Unique, parseIdx.Type, parseIdx.Expressions, parseIdx.Visible, parseIdx.Comment)- }-}--func compareForeignKeys(t *testing.T, tableName string, syncFKs, parseFKs []*storepb.ForeignKeyMetadata) {- require.Equal(t, len(syncFKs), len(parseFKs), "table %s: number of foreign keys should match", tableName)-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.ForeignKeyMetadata)- for _, fk := range syncFKs {- syncMap[fk.Name] = fk- }-- for _, parseFk := range parseFKs {- syncFk, exists := syncMap[parseFk.Name]- require.True(t, exists, "table %s: foreign key %s should exist in sync metadata", tableName, parseFk.Name)-- require.ElementsMatch(t, syncFk.Columns, parseFk.Columns, "table %s, FK %s: columns should match", tableName, parseFk.Name)- require.Equal(t, syncFk.ReferencedTable, parseFk.ReferencedTable, "table %s, FK %s: referenced table should match", tableName, parseFk.Name)- require.ElementsMatch(t, syncFk.ReferencedColumns, parseFk.ReferencedColumns, "table %s, FK %s: referenced columns should match", tableName, parseFk.Name)- }-}--func comparePartitions(t *testing.T, tableName string, syncParts, parseParts []*storepb.TablePartitionMetadata) {- require.Equal(t, len(syncParts), len(parseParts), "table %s: number of partitions should match", tableName)-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.TablePartitionMetadata)- for _, part := range syncParts {- syncMap[part.Name] = part- }-- for _, parsePart := range parseParts {- syncPart, exists := syncMap[parsePart.Name]- require.True(t, exists, "table %s: partition %s should exist in sync metadata", tableName, parsePart.Name)- // Use AST-based semantic comparison for partition expressions- equal := ast.CompareExpressionsSemantically(syncPart.Expression, parsePart.Expression)- require.True(t, equal, "table %s, partition %s: expression should be semantically equivalent. sync: %q, parse: %q",- tableName, parsePart.Name, syncPart.Expression, parsePart.Expression)- require.Equal(t, syncPart.Value, parsePart.Value, "table %s, partition %s: value should match", tableName, parsePart.Name)- }-}--func compareViews(t *testing.T, syncViews, parseViews []*storepb.ViewMetadata) {- require.Equal(t, len(syncViews), len(parseViews), "number of views should match")-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.ViewMetadata)- for _, view := range syncViews {- syncMap[view.Name] = view- }-- for _, parseView := range parseViews {- syncView, exists := syncMap[parseView.Name]- require.True(t, exists, "view %s should exist in sync metadata", parseView.Name)-- // Compare view definitions using PostgreSQL view comparer for better normalization- comparer := &PostgreSQLViewComparer{}- definitionsEqual := comparer.compareViewsSemanticaly(syncView.Definition, parseView.Definition)- require.True(t, definitionsEqual, "view %s: definition should match\nSync: %s\nParse: %s", parseView.Name, syncView.Definition, parseView.Definition)-- // Compare comment if present- if syncView.Comment != "" || parseView.Comment != "" {- require.Equal(t, syncView.Comment, parseView.Comment, "view %s: comment should match", parseView.Name)- }-- // Compare rules- compareRules(t, parseView.Name, syncView.Rules, parseView.Rules)- }-}--func compareFunctions(t *testing.T, syncFuncs, parseFuncs []*storepb.FunctionMetadata) {- // Function comparison is tricky because signatures might be formatted differently- t.Logf("sync has %d functions, parse has %d functions", len(syncFuncs), len(parseFuncs))-- // Currently the parser doesn't extract functions from DDL, so we expect 0 functions from parser- // If parser starts extracting functions, we should implement proper comparison here- if len(parseFuncs) == 0 && len(syncFuncs) > 0 {- // This is expected - parser doesn't extract functions yet- return- }-- // If parser starts returning functions, implement full comparison- require.Equal(t, len(syncFuncs), len(parseFuncs), "number of functions should match")-- // Create maps for easier comparison - use function signature for mapping- syncMap := make(map[string]*storepb.FunctionMetadata)- for _, fn := range syncFuncs {- syncMap[fn.Signature] = fn- }-- for _, parseFn := range parseFuncs {- // Try to find matching function by signature- var syncFn *storepb.FunctionMetadata- for _, sf := range syncFuncs {- if normalizeSignature(sf.Signature) == normalizeSignature(parseFn.Signature) {- syncFn = sf- break- }- }-- require.NotNil(t, syncFn, "function with signature %s should exist in sync metadata", parseFn.Signature)-- // Compare function definitions using PostgreSQL function comparer- comparer := &PostgreSQLFunctionComparer{}- syncFunc := &storepb.FunctionMetadata{Definition: syncFn.Definition}- parseFunc := &storepb.FunctionMetadata{Definition: parseFn.Definition}-- // Use function comparer to check if they are equal- if !comparer.Equal(syncFunc, parseFunc) {- // Get detailed comparison for better error message- result, err := comparer.CompareDetailed(syncFunc, parseFunc)- if err != nil {- t.Errorf("function %s: error comparing functions: %v", parseFn.Name, err)- } else if result != nil {- t.Errorf("function %s: definitions differ - SignatureChanged: %v, BodyChanged: %v, AttributesChanged: %v",- parseFn.Name, result.SignatureChanged, result.BodyChanged, result.AttributesChanged)- t.Logf(" Sync definition: %s", syncFn.Definition)- t.Logf(" Parse definition: %s", parseFn.Definition)- }- }-- // Compare comment if present- if syncFn.Comment != "" || parseFn.Comment != "" {- require.Equal(t, syncFn.Comment, parseFn.Comment, "function %s: comment should match", parseFn.Name)- }- }-}--func compareSequences(t *testing.T, syncSeqs, parseSeqs []*storepb.SequenceMetadata) {- // Create maps for easier comparison- syncMap := make(map[string]*storepb.SequenceMetadata)- for _, seq := range syncSeqs {- syncMap[seq.Name] = seq- }-- parseMap := make(map[string]*storepb.SequenceMetadata)- for _, seq := range parseSeqs {- parseMap[seq.Name] = seq- }-- // Check sequences in parseSeqs- for _, parseSeq := range parseSeqs {- syncSeq, exists := syncMap[parseSeq.Name]- if !exists {- // SERIAL columns create implicit sequences that might not be in DDL- t.Logf("sequence %s exists in parse but not in sync (might be implicit from SERIAL)", parseSeq.Name)- continue- }-- // Compare basic properties- if parseSeq.Start != "" {- require.Equal(t, syncSeq.Start, parseSeq.Start, "sequence %s: start value should match", parseSeq.Name)- }- if parseSeq.Increment != "" {- require.Equal(t, syncSeq.Increment, parseSeq.Increment, "sequence %s: increment should match", parseSeq.Name)- }- }-- // Check sequences in syncSeqs that are not in parseSeqs- for _, syncSeq := range syncSeqs {- _, exists := parseMap[syncSeq.Name]- if !exists {- // Skip implicit sequences created by SERIAL columns- if strings.Contains(syncSeq.Name, "_id_seq") || strings.Contains(syncSeq.Name, "_seq") {- t.Logf("sequence %s exists in sync but not in parse (implicit sequence from SERIAL column)", syncSeq.Name)- continue- }- // For explicitly created sequences, this is an error- require.True(t, exists, "sequence %s should exist in parsed metadata", syncSeq.Name)- }- }-}--func compareEnums(t *testing.T, syncEnums, parseEnums []*storepb.EnumTypeMetadata) {- require.Equal(t, len(syncEnums), len(parseEnums), "number of enums should match")-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.EnumTypeMetadata)- for _, enum := range syncEnums {- syncMap[enum.Name] = enum- }-- for _, parseEnum := range parseEnums {- syncEnum, exists := syncMap[parseEnum.Name]- require.True(t, exists, "enum %s should exist in sync metadata", parseEnum.Name)- require.ElementsMatch(t, syncEnum.Values, parseEnum.Values, "enum %s: values should match", parseEnum.Name)- }-}--func compareExtensions(t *testing.T, syncExts, parseExts []*storepb.ExtensionMetadata) {- // Create maps for easier comparison- syncMap := make(map[string]*storepb.ExtensionMetadata)- for _, ext := range syncExts {- syncMap[ext.Name] = ext- }-- for _, parseExt := range parseExts {- syncExt, exists := syncMap[parseExt.Name]- require.True(t, exists, "extension %s should exist in sync metadata", parseExt.Name)- require.Equal(t, syncExt.Schema, parseExt.Schema, "extension %s: schema should match", parseExt.Name)- }-}--func compareMaterializedViews(t *testing.T, syncMViews, parseMViews []*storepb.MaterializedViewMetadata) {- // Materialized views are not currently supported by the parser- // The parser may incorrectly classify them as tables, so we handle this gracefully- if len(parseMViews) == 0 && len(syncMViews) > 0 {- t.Logf("Parser doesn't extract materialized views yet - found %d in sync", len(syncMViews))- return- }-- require.Equal(t, len(syncMViews), len(parseMViews), "number of materialized views should match")-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.MaterializedViewMetadata)- for _, mv := range syncMViews {- syncMap[mv.Name] = mv- }-- for _, parseMv := range parseMViews {- syncMv, exists := syncMap[parseMv.Name]- require.True(t, exists, "materialized view %s should exist in sync metadata", parseMv.Name)-- // Compare definitions using PostgreSQL view comparer for better normalization- comparer := &PostgreSQLViewComparer{}- definitionsEqual := comparer.compareViewsSemanticaly(syncMv.Definition, parseMv.Definition)- require.True(t, definitionsEqual, "materialized view %s: definition should match\nSync: %s\nParse: %s", parseMv.Name, syncMv.Definition, parseMv.Definition)-- // Compare comment if present- if syncMv.Comment != "" || parseMv.Comment != "" {- require.Equal(t, syncMv.Comment, parseMv.Comment, "materialized view %s: comment should match", parseMv.Name)- }-- // Compare indexes on materialized views if present- if len(syncMv.Indexes) > 0 || len(parseMv.Indexes) > 0 {- compareIndexes(t, parseMv.Name, syncMv.Indexes, parseMv.Indexes)- }-- // Compare triggers on materialized views if present- if len(syncMv.Triggers) > 0 || len(parseMv.Triggers) > 0 {- compareTriggers(t, parseMv.Name, syncMv.Triggers, parseMv.Triggers)- }- }-}--func compareProcedures(t *testing.T, syncProcs, parseProcs []*storepb.ProcedureMetadata) {- // Procedures might not be extracted by parser yet, so handle gracefully- if len(parseProcs) == 0 && len(syncProcs) > 0 {- t.Logf("Parser doesn't extract procedures yet - found %d in sync", len(syncProcs))- return- }-- require.Equal(t, len(syncProcs), len(parseProcs), "number of procedures should match")-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.ProcedureMetadata)- for _, proc := range syncProcs {- syncMap[proc.Name] = proc- }-- for _, parseProc := range parseProcs {- syncProc, exists := syncMap[parseProc.Name]- require.True(t, exists, "procedure %s should exist in sync metadata", parseProc.Name)-- // Compare definitions- syncDef := normalizeSQL(syncProc.Definition)- parseDef := normalizeSQL(parseProc.Definition)- require.Equal(t, syncDef, parseDef, "procedure %s: definition should match", parseProc.Name)- }-}--func compareRules(t *testing.T, objectName string, syncRules, parseRules []*storepb.RuleMetadata) {- // Rules might not be extracted by parser yet, so handle gracefully- if len(parseRules) == 0 && len(syncRules) > 0 {- t.Logf("Object %s: Parser doesn't extract rules yet - found %d in sync", objectName, len(syncRules))- return- }-- require.Equal(t, len(syncRules), len(parseRules), "object %s: number of rules should match", objectName)-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.RuleMetadata)- for _, rule := range syncRules {- syncMap[rule.Name] = rule- }-- for _, parseRule := range parseRules {- syncRule, exists := syncMap[parseRule.Name]- require.True(t, exists, "object %s: rule %s should exist in sync metadata", objectName, parseRule.Name)-- // Compare rule properties- require.Equal(t, syncRule.Event, parseRule.Event, "object %s: rule %s: event should match", objectName, parseRule.Name)- require.Equal(t, syncRule.IsInstead, parseRule.IsInstead, "object %s: rule %s: is_instead should match", objectName, parseRule.Name)- require.Equal(t, syncRule.IsEnabled, parseRule.IsEnabled, "object %s: rule %s: is_enabled should match", objectName, parseRule.Name)-- // Compare conditions (might be empty)- if syncRule.Condition != "" || parseRule.Condition != "" {- require.Equal(t, syncRule.Condition, parseRule.Condition, "object %s: rule %s: condition should match", objectName, parseRule.Name)- }-- // Compare definitions - normalize them first- syncDef := normalizeSQL(syncRule.Definition)- parseDef := normalizeSQL(parseRule.Definition)- require.Equal(t, syncDef, parseDef, "object %s: rule %s: definition should match", objectName, parseRule.Name)- }-}--func compareTriggers(t *testing.T, tableName string, syncTriggers, parseTriggers []*storepb.TriggerMetadata) {- // Triggers might not be extracted by parser yet, so handle gracefully- if len(parseTriggers) == 0 && len(syncTriggers) > 0 {- t.Logf("Table %s: Parser doesn't extract triggers yet - found %d in sync", tableName, len(syncTriggers))- return- }-- require.Equal(t, len(syncTriggers), len(parseTriggers), "table %s: number of triggers should match", tableName)-- // Create maps for easier comparison- syncMap := make(map[string]*storepb.TriggerMetadata)- for _, trigger := range syncTriggers {- syncMap[trigger.Name] = trigger- }-- for _, parseTrigger := range parseTriggers {- syncTrigger, exists := syncMap[parseTrigger.Name]- require.True(t, exists, "table %s: trigger %s should exist in sync metadata", tableName, parseTrigger.Name)-- // Compare basic trigger properties- require.Equal(t, syncTrigger.Event, parseTrigger.Event, "table %s, trigger %s: event should match", tableName, parseTrigger.Name)- require.Equal(t, syncTrigger.Timing, parseTrigger.Timing, "table %s, trigger %s: timing should match", tableName, parseTrigger.Name)-- // Compare trigger body/definition if available- if syncTrigger.Body != "" || parseTrigger.Body != "" {- syncBody := normalizeSQL(syncTrigger.Body)- parseBody := normalizeSQL(parseTrigger.Body)- require.Equal(t, syncBody, parseBody, "table %s, trigger %s: body should match", tableName, parseTrigger.Name)- }- }-}--func compareAllSchemas(t *testing.T, syncSchemas, parseSchemas []*storepb.SchemaMetadata) {- // For cross-schema test cases, we need to compare schemas beyond just 'public'- // Create maps for easier comparison- syncMap := make(map[string]*storepb.SchemaMetadata)- for _, schema := range syncSchemas {- syncMap[schema.Name] = schema- }-- parseMap := make(map[string]*storepb.SchemaMetadata)- for _, schema := range parseSchemas {- parseMap[schema.Name] = schema- }-- // Check that important schemas exist in both- for _, parseSchema := range parseSchemas {- if parseSchema.Name == "information_schema" || parseSchema.Name == "pg_catalog" {- // Skip system schemas- continue- }-- syncSchema, exists := syncMap[parseSchema.Name]- if !exists {- t.Logf("Schema %s exists in parse but not in sync (might be expected for some test cases)", parseSchema.Name)- continue- }-- // Compare schema-specific content- t.Logf("Comparing schema: %s", parseSchema.Name)-- // Compare tables in this schema- compareTables(t, syncSchema.Tables, parseSchema.Tables)-- // Compare views in this schema- compareViews(t, syncSchema.Views, parseSchema.Views)-- // Compare functions in this schema- compareFunctions(t, syncSchema.Functions, parseSchema.Functions)- }-}--// validateWithSchemaDiffer validates that the schema differ returns no significant differences between sync and parse metadata-func validateWithSchemaDiffer(t *testing.T, testName string, syncMeta, parseMeta *storepb.DatabaseSchemaMetadata) {- // Convert metadata to model.DatabaseMetadata for differ- syncSchema := model.NewDatabaseMetadata(syncMeta, nil, nil, storepb.Engine_POSTGRES, false)- parseSchema := model.NewDatabaseMetadata(parseMeta, nil, nil, storepb.Engine_POSTGRES, false)-- // Get schema diff- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, syncSchema, parseSchema)- require.NoError(t, err, "test case %s: schema differ should not error", testName)-- if diff == nil {- // No diff is expected - this is good- return- }-- // Check that all diff categories are empty - DDL parser should fully match PostgreSQL behavior- var diffMessages []string-- if len(diff.SchemaChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d schema changes", len(diff.SchemaChanges)))- for _, change := range diff.SchemaChanges {- t.Logf("Schema change: %s %s", change.Action, change.SchemaName)- }- }-- if len(diff.TableChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d table changes", len(diff.TableChanges)))- for _, change := range diff.TableChanges {- t.Logf("Table change: %s %s.%s", change.Action, change.SchemaName, change.TableName)- // Log detailed changes within the table- if len(change.ColumnChanges) > 0 {- t.Logf(" Column changes: %d", len(change.ColumnChanges))- for _, colChange := range change.ColumnChanges {- if colChange.OldColumn != nil && colChange.NewColumn != nil {- t.Logf(" %s column %s: %s -> %s", colChange.Action, colChange.NewColumn.Name, colChange.OldColumn.Type, colChange.NewColumn.Type)- if colChange.OldColumn.Default != colChange.NewColumn.Default {- t.Logf(" Default changed: '%s' -> '%s'", colChange.OldColumn.Default, colChange.NewColumn.Default)- }- if colChange.OldColumn.Nullable != colChange.NewColumn.Nullable {- t.Logf(" Nullable changed: %t -> %t", colChange.OldColumn.Nullable, colChange.NewColumn.Nullable)- }- } else if colChange.NewColumn != nil {- t.Logf(" %s column %s: %s", colChange.Action, colChange.NewColumn.Name, colChange.NewColumn.Type)- } else if colChange.OldColumn != nil {- t.Logf(" %s column %s: %s", colChange.Action, colChange.OldColumn.Name, colChange.OldColumn.Type)- }- }- }- if len(change.IndexChanges) > 0 {- t.Logf(" Index changes: %d", len(change.IndexChanges))- for i, idxChange := range change.IndexChanges {- // Look for DROP/CREATE pairs that indicate a difference was detected- if i+1 < len(change.IndexChanges) && idxChange.Action == "DROP" && change.IndexChanges[i+1].Action == "CREATE" &&- idxChange.OldIndex.Name == change.IndexChanges[i+1].NewIndex.Name {- // This is a DROP/CREATE pair - log detailed comparison- oldIdx := idxChange.OldIndex- newIdx := change.IndexChanges[i+1].NewIndex- t.Logf(" DETAILED INDEX COMPARISON for %s (DROP/CREATE pair):", oldIdx.Name)- t.Logf(" Type: old='%s' vs new='%s' (equal: %v)", oldIdx.Type, newIdx.Type, oldIdx.Type == newIdx.Type)- t.Logf(" Unique: old=%v vs new=%v (equal: %v)", oldIdx.Unique, newIdx.Unique, oldIdx.Unique == newIdx.Unique)- t.Logf(" Primary: old=%v vs new=%v (equal: %v)", oldIdx.Primary, newIdx.Primary, oldIdx.Primary == newIdx.Primary)- t.Logf(" Expressions: old=%v vs new=%v (equal: %v)", oldIdx.Expressions, newIdx.Expressions, fmt.Sprintf("%v", oldIdx.Expressions) == fmt.Sprintf("%v", newIdx.Expressions))- t.Logf(" KeyLength: old=%v vs new=%v (equal: %v)", oldIdx.KeyLength, newIdx.KeyLength, fmt.Sprintf("%v", oldIdx.KeyLength) == fmt.Sprintf("%v", newIdx.KeyLength))- t.Logf(" Descending: old=%v vs new=%v (equal: %v)", oldIdx.Descending, newIdx.Descending, fmt.Sprintf("%v", oldIdx.Descending) == fmt.Sprintf("%v", newIdx.Descending))- t.Logf(" Visible: old=%v vs new=%v (equal: %v)", oldIdx.Visible, newIdx.Visible, oldIdx.Visible == newIdx.Visible)- t.Logf(" Definition: old='%s' vs new='%s' (equal: %v)", oldIdx.Definition, newIdx.Definition, oldIdx.Definition == newIdx.Definition)- t.Logf(" IsConstraint: old=%v vs new=%v (equal: %v)", oldIdx.IsConstraint, newIdx.IsConstraint, oldIdx.IsConstraint == newIdx.IsConstraint)- }-- if idxChange.NewIndex != nil && idxChange.OldIndex != nil {- // Log detailed comparison for ALTER cases- t.Logf(" DETAILED INDEX COMPARISON for %s:", idxChange.NewIndex.Name)- t.Logf(" Type: old='%s' vs new='%s' (equal: %v)", idxChange.OldIndex.Type, idxChange.NewIndex.Type, idxChange.OldIndex.Type == idxChange.NewIndex.Type)- t.Logf(" Unique: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Unique, idxChange.NewIndex.Unique, idxChange.OldIndex.Unique == idxChange.NewIndex.Unique)- t.Logf(" Primary: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Primary, idxChange.NewIndex.Primary, idxChange.OldIndex.Primary == idxChange.NewIndex.Primary)- t.Logf(" Expressions: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Expressions, idxChange.NewIndex.Expressions, fmt.Sprintf("%v", idxChange.OldIndex.Expressions) == fmt.Sprintf("%v", idxChange.NewIndex.Expressions))- t.Logf(" KeyLength: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.KeyLength, idxChange.NewIndex.KeyLength, fmt.Sprintf("%v", idxChange.OldIndex.KeyLength) == fmt.Sprintf("%v", idxChange.NewIndex.KeyLength))- t.Logf(" Descending: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Descending, idxChange.NewIndex.Descending, fmt.Sprintf("%v", idxChange.OldIndex.Descending) == fmt.Sprintf("%v", idxChange.NewIndex.Descending))- t.Logf(" Visible: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Visible, idxChange.NewIndex.Visible, idxChange.OldIndex.Visible == idxChange.NewIndex.Visible)- t.Logf(" Definition: old='%s' vs new='%s' (equal: %v)", idxChange.OldIndex.Definition, idxChange.NewIndex.Definition, idxChange.OldIndex.Definition == idxChange.NewIndex.Definition)- }-- if idxChange.NewIndex != nil {- t.Logf(" %s index %s: type=%s, unique=%v, primary=%v, expressions=%v, definition=%s",- idxChange.Action, idxChange.NewIndex.Name, idxChange.NewIndex.Type,- idxChange.NewIndex.Unique, idxChange.NewIndex.Primary, idxChange.NewIndex.Expressions, idxChange.NewIndex.Definition)- } else if idxChange.OldIndex != nil {- t.Logf(" %s index %s: type=%s, unique=%v, primary=%v, expressions=%v, definition=%s",- idxChange.Action, idxChange.OldIndex.Name, idxChange.OldIndex.Type,- idxChange.OldIndex.Unique, idxChange.OldIndex.Primary, idxChange.OldIndex.Expressions, idxChange.OldIndex.Definition)- }- }- }- if len(change.ForeignKeyChanges) > 0 {- t.Logf(" Foreign key changes: %d", len(change.ForeignKeyChanges))- for _, fkChange := range change.ForeignKeyChanges {- if fkChange.NewForeignKey != nil {- t.Logf(" %s FK %s: columns=%v -> %s.%s(%v), onDelete=%s",- fkChange.Action, fkChange.NewForeignKey.Name, fkChange.NewForeignKey.Columns,- fkChange.NewForeignKey.ReferencedSchema, fkChange.NewForeignKey.ReferencedTable,- fkChange.NewForeignKey.ReferencedColumns, fkChange.NewForeignKey.OnDelete)- } else if fkChange.OldForeignKey != nil {- t.Logf(" %s FK %s: columns=%v -> %s.%s(%v), onDelete=%s",- fkChange.Action, fkChange.OldForeignKey.Name, fkChange.OldForeignKey.Columns,- fkChange.OldForeignKey.ReferencedSchema, fkChange.OldForeignKey.ReferencedTable,- fkChange.OldForeignKey.ReferencedColumns, fkChange.OldForeignKey.OnDelete)- }- }- }- if len(change.CheckConstraintChanges) > 0 {- t.Logf(" Check constraint changes: %d", len(change.CheckConstraintChanges))- for _, checkChange := range change.CheckConstraintChanges {- if checkChange.NewCheckConstraint != nil && checkChange.OldCheckConstraint != nil {- t.Logf(" %s CHECK %s: old='%s' -> new='%s'",- checkChange.Action, checkChange.NewCheckConstraint.Name,- checkChange.OldCheckConstraint.Expression, checkChange.NewCheckConstraint.Expression)- } else if checkChange.NewCheckConstraint != nil {- t.Logf(" %s CHECK %s: expression='%s'",- checkChange.Action, checkChange.NewCheckConstraint.Name, checkChange.NewCheckConstraint.Expression)- } else if checkChange.OldCheckConstraint != nil {- t.Logf(" %s CHECK %s: expression='%s'",- checkChange.Action, checkChange.OldCheckConstraint.Name, checkChange.OldCheckConstraint.Expression)- }- }- }- if len(change.TriggerChanges) > 0 {- t.Logf(" Trigger changes: %d", len(change.TriggerChanges))- for _, triggerChange := range change.TriggerChanges {- if triggerChange.NewTrigger != nil && triggerChange.OldTrigger != nil {- t.Logf(" %s TRIGGER %s: old='%s' -> new='%s'",- triggerChange.Action, triggerChange.NewTrigger.Name,- triggerChange.OldTrigger.Body, triggerChange.NewTrigger.Body)- } else if triggerChange.NewTrigger != nil {- t.Logf(" %s TRIGGER %s: body='%s'",- triggerChange.Action, triggerChange.NewTrigger.Name, triggerChange.NewTrigger.Body)- } else if triggerChange.OldTrigger != nil {- t.Logf(" %s TRIGGER %s: body='%s'",- triggerChange.Action, triggerChange.OldTrigger.Name, triggerChange.OldTrigger.Body)- }- }- }- }- }-- if len(diff.ViewChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d view changes", len(diff.ViewChanges)))- for _, change := range diff.ViewChanges {- t.Logf("View change: %s %s.%s", change.Action, change.SchemaName, change.ViewName)- }- }-- if len(diff.MaterializedViewChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d materialized view changes", len(diff.MaterializedViewChanges)))- for _, change := range diff.MaterializedViewChanges {- t.Logf("Materialized view change: %s %s.%s", change.Action, change.SchemaName, change.MaterializedViewName)- }- }-- if len(diff.FunctionChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d function changes", len(diff.FunctionChanges)))- for _, change := range diff.FunctionChanges {- t.Logf("Function change: %s %s.%s", change.Action, change.SchemaName, change.FunctionName)- // Log detailed function comparison for debugging- if change.OldFunction != nil && change.NewFunction != nil {- t.Logf(" Old function signature: %s", change.OldFunction.Signature)- t.Logf(" New function signature: %s", change.NewFunction.Signature)- t.Logf(" Old function definition: %s", change.OldFunction.Definition)- t.Logf(" New function definition: %s", change.NewFunction.Definition)- t.Logf(" Definitions equal: %v", change.OldFunction.Definition == change.NewFunction.Definition)- t.Logf(" Signatures equal: %v", change.OldFunction.Signature == change.NewFunction.Signature)- }- }- }-- if len(diff.ProcedureChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d procedure changes", len(diff.ProcedureChanges)))- for _, change := range diff.ProcedureChanges {- t.Logf("Procedure change: %s %s.%s", change.Action, change.SchemaName, change.ProcedureName)- }- }-- if len(diff.SequenceChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d sequence changes", len(diff.SequenceChanges)))- for _, change := range diff.SequenceChanges {- t.Logf("Sequence change: %s %s.%s", change.Action, change.SchemaName, change.SequenceName)- if change.OldSequence != nil && change.NewSequence == nil {- t.Logf(" ISSUE: SERIAL sequence %s missing in parsed metadata", change.SequenceName)- } else if change.NewSequence != nil && change.OldSequence == nil {- t.Logf(" ISSUE: Unexpected sequence %s in parsed metadata", change.SequenceName)- }- }- }-- if len(diff.EnumTypeChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d enum type changes", len(diff.EnumTypeChanges)))- for _, change := range diff.EnumTypeChanges {- t.Logf("Enum type change: %s %s.%s", change.Action, change.SchemaName, change.EnumTypeName)- }- }-- if len(diff.EventChanges) > 0 {- diffMessages = append(diffMessages, fmt.Sprintf("%d event changes", len(diff.EventChanges)))- for _, change := range diff.EventChanges {- t.Logf("Event change: %s %s", change.Action, change.EventName)- }- }-- // If we have any differences, fail the test - DDL parser must match PostgreSQL exactly- if len(diffMessages) > 0 {- require.Fail(t, fmt.Sprintf("test case %s: DDL parser should fully replicate PostgreSQL behavior. Differences found: %s", testName, strings.Join(diffMessages, ", ")))- }-}--// validateRoutingRules validates that rules are handled correctly for all test cases-func validateRoutingRules(t *testing.T, metadata *storepb.DatabaseSchemaMetadata) {- // This validation ensures:- // 1. No cycle errors occur with rules (if we got here, it passed)- // 2. GetDatabaseDefinition works without errors- // 3. Rules are properly included in the output-- require.NotNil(t, metadata, "Should have metadata without cycle errors")-- // Test that GetDatabaseDefinition works without cycle errors- ctx := schema.GetDefinitionContext{- PrintHeader: false,- }- definition, err := GetDatabaseDefinition(ctx, metadata)- require.NoError(t, err, "GetDatabaseDefinition should not error even with rules")- require.NotEmpty(t, definition)-- // Count rules across all schemas, tables, and views- var totalRules int- var hasRules bool-- for _, schema := range metadata.Schemas {- // Check tables for rules- for _, table := range schema.Tables {- if len(table.Rules) > 0 {- hasRules = true- totalRules += len(table.Rules)-- // Verify each rule is in the output- for _, rule := range table.Rules {- require.NotEmpty(t, rule.Definition, "Rule should have definition")- require.NotEmpty(t, rule.Event, "Rule should have event type")- // Verify the rule appears in the output- if !strings.Contains(definition, rule.Name) {- t.Logf("Warning: Rule %s not found in output (might be a system rule)", rule.Name)- }- }- }- }-- // Check views for rules- for _, view := range schema.Views {- if len(view.Rules) > 0 {- hasRules = true- totalRules += len(view.Rules)-- // Verify each rule is in the output- for _, rule := range view.Rules {- require.NotEmpty(t, rule.Definition, "Rule should have definition")- require.NotEmpty(t, rule.Event, "Rule should have event type")- // Non-SELECT rules should be in the output- if rule.Event != "SELECT" && !strings.Contains(definition, rule.Name) {- t.Logf("Warning: Rule %s not found in output", rule.Name)- }- }- }- }- }-- // Log summary for debugging- if hasRules {- t.Logf("✓ Found %d rules in metadata, GetDatabaseDefinition succeeded without cycles", totalRules)-- // For test cases with specific routing patterns, do additional validation- // Check for the routing rules pattern (view with INSERT/UPDATE/DELETE rules)- for _, schema := range metadata.Schemas {- for _, view := range schema.Views {- if view.Name == "new_name" && len(view.Rules) > 0 {- // This is the routing pattern - verify correct ordering- tablePos := strings.Index(definition, `CREATE TABLE "public"."old_name"`)- viewPos := strings.Index(definition, `CREATE VIEW "public"."new_name"`)- if tablePos > -1 && viewPos > -1 {- require.Less(t, tablePos, viewPos, "With routing rules: table should come before view due to dependency")- t.Log("✓ Routing rules pattern validated - correct ordering maintained")- }- }- }- }- } else {- t.Log("No rules in this test case - validation passed")- }-}--// compareIndexDefinitionsSemanticaly compares two PostgreSQL index definitions using AST-based semantic comparison-func compareIndexDefinitionsSemanticaly(def1, def2 string) bool {- if strings.TrimSpace(def1) == strings.TrimSpace(def2) {- return true- }-- // Parse the index definitions to extract components- parts1 := parseIndexDefinition(def1)- parts2 := parseIndexDefinition(def2)-- if parts1 == nil || parts2 == nil {- // Fallback to case-insensitive string comparison if parsing fails- return strings.EqualFold(strings.TrimSpace(def1), strings.TrimSpace(def2))- }-- // Compare non-expression parts- if !strings.EqualFold(parts1.CreateClause, parts2.CreateClause) ||- !strings.EqualFold(parts1.IndexName, parts2.IndexName) ||- !strings.EqualFold(parts1.OnClause, parts2.OnClause) ||- !strings.EqualFold(parts1.UsingClause, parts2.UsingClause) {- return false- }-- // Compare expressions using semantic comparison- comparer := ast.NewPostgreSQLExpressionComparer()- if parts1.ExpressionsClause != "" && parts2.ExpressionsClause != "" {- equal, err := comparer.CompareExpressions(parts1.ExpressionsClause, parts2.ExpressionsClause)- if err != nil {- // Fallback to string comparison if expression comparison fails- return strings.EqualFold(strings.TrimSpace(parts1.ExpressionsClause), strings.TrimSpace(parts2.ExpressionsClause))- }- if !equal {- return false- }- } else if parts1.ExpressionsClause != parts2.ExpressionsClause {- return false- }-- // Compare WHERE clause using semantic comparison- if parts1.WhereClause != "" && parts2.WhereClause != "" {- equal, err := comparer.CompareExpressions(parts1.WhereClause, parts2.WhereClause)- if err != nil {- // Fallback to string comparison if expression comparison fails- return strings.EqualFold(strings.TrimSpace(parts1.WhereClause), strings.TrimSpace(parts2.WhereClause))- }- if !equal {- return false- }- } else if parts1.WhereClause != parts2.WhereClause {- return false- }-- // Compare optional clauses- return strings.EqualFold(parts1.IncludeClause, parts2.IncludeClause) &&- strings.EqualFold(parts1.WithClause, parts2.WithClause)-}--// indexDefinitionParts represents the parsed components of an index definition-type indexDefinitionParts struct {- CreateClause string // "CREATE [UNIQUE] INDEX"- IndexName string // index name- OnClause string // "ON table_name"- UsingClause string // "USING method"- ExpressionsClause string // column list or expressions inside parentheses- IncludeClause string // "INCLUDE (...)" clause- WhereClause string // WHERE condition (without WHERE keyword)- WithClause string // WITH (...) clause-}--// parseIndexDefinition parses a PostgreSQL index definition into its components-func parseIndexDefinition(definition string) *indexDefinitionParts {- def := strings.TrimSpace(definition)- if def == "" {- return nil- }-- // Remove trailing semicolon- def = strings.TrimSuffix(def, ";")-- parts := &indexDefinitionParts{}-- // Split by major keywords to identify parts- lowerDef := strings.ToLower(def)-- // Find CREATE clause- createEnd := strings.Index(lowerDef, " on ")- if createEnd == -1 {- return nil- }-- // Extract index name from CREATE clause- createPart := def[:createEnd]- parts.CreateClause = createPart-- // Extract index name (last word before "on")- words := strings.Fields(createPart)- if len(words) > 0 {- parts.IndexName = words[len(words)-1]- }-- remaining := def[createEnd:]- lowerRemaining := strings.ToLower(remaining)-- // Find ON clause- usingPos := strings.Index(lowerRemaining, " using ")- if usingPos == -1 {- return nil- }- parts.OnClause = strings.TrimSpace(remaining[:usingPos])-- remaining = remaining[usingPos:]-- // Find USING clause- parenPos := strings.Index(remaining, "(")- if parenPos == -1 {- return nil- }- parts.UsingClause = strings.TrimSpace(remaining[:parenPos])-- remaining = remaining[parenPos:]-- // Extract expressions (content within first set of parentheses)- parenCount := 0- exprEnd := -1- for i, r := range remaining {- if r == '(' {- parenCount++- } else if r == ')' {- parenCount--- if parenCount == 0 {- exprEnd = i- break- }- }- }-- if exprEnd == -1 {- return nil- }-- // Extract content within parentheses (excluding the parentheses themselves)- parts.ExpressionsClause = strings.TrimSpace(remaining[1:exprEnd])-- if len(remaining) > exprEnd+1 {- remaining = strings.TrimSpace(remaining[exprEnd+1:])- lowerRemaining = strings.ToLower(remaining)-- // Look for optional clauses in order: INCLUDE, WHERE, WITH-- // INCLUDE clause- if strings.HasPrefix(lowerRemaining, "include") {- includeEnd := strings.Index(remaining, ")")- if includeEnd != -1 {- parts.IncludeClause = strings.TrimSpace(remaining[:includeEnd+1])- if len(remaining) > includeEnd+1 {- remaining = strings.TrimSpace(remaining[includeEnd+1:])- lowerRemaining = strings.ToLower(remaining)- } else {- remaining = ""- lowerRemaining = ""- }- }- }-- // WHERE clause- if strings.HasPrefix(lowerRemaining, "where ") {- whereStart := 6 // length of "where "- // Find the end of WHERE clause (could be end of string or start of WITH clause)- withPos := strings.Index(lowerRemaining, " with ")- var whereEnd int- if withPos != -1 {- whereEnd = withPos- } else {- whereEnd = len(remaining)- }- parts.WhereClause = strings.TrimSpace(remaining[whereStart:whereEnd])-- if withPos != -1 {- remaining = strings.TrimSpace(remaining[withPos:])- lowerRemaining = strings.ToLower(remaining)- } else {- remaining = ""- lowerRemaining = ""- }- }-- // WITH clause- if strings.HasPrefix(lowerRemaining, "with ") {- parts.WithClause = strings.TrimSpace(remaining)- }- }-- return parts-}--// validateArrayTypes validates that array types are correctly mapped to their PostgreSQL internal representations-func validateArrayTypes(t *testing.T, syncMetadata, parseMetadata *storepb.DatabaseSchemaMetadata) {- t.Helper()-- // Expected array type mappings based on our implementation- expectedArrayTypes := map[string]string{- // Integer arrays (all int variants should map to _int4)- "int_array": "_int4",- "int4_array": "_int4",- "integer_array": "_int4",- "bigint_array": "_int8",- "int8_array": "_int8",- "smallint_array": "_int2",- "int2_array": "_int2",-- // Multi-dimensional arrays (should be same as single dimension)- "int_multi_array": "_int4",- "text_multi_array": "_text",-- // Floating point arrays- "real_array": "_float4",- "float4_array": "_float4",- "double_precision_array": "_float8",- "float8_array": "_float8",-- // Character arrays (all should map to appropriate internal types)- "char_array": "_bpchar",- "char_with_length": "_bpchar",- "varchar_array": "_varchar",- "varchar_with_length": "_varchar",- "character_array": "_bpchar",- "character_with_length": "_bpchar",- "character_varying_array": "_varchar",- "character_varying_with_length": "_varchar",- "text_array": "_text",-- // Boolean arrays- "bool_array": "_bool",- "boolean_array": "_bool",-- // Numeric arrays- "numeric_array": "_numeric",- "numeric_with_precision": "_numeric",- "decimal_array": "_numeric",- "decimal_with_precision": "_numeric",- "money_array": "_money",-- // Date/time arrays- "date_array": "_date",- "time_array": "_time",- "time_with_tz": "_timetz",- "timetz_array": "_timetz",- "timestamp_array": "_timestamp",- "timestamp_with_tz": "_timestamptz",- "timestamptz_array": "_timestamptz",- "interval_array": "_interval",-- // Binary and UUID arrays- "bytea_array": "_bytea",- "uuid_array": "_uuid",-- // JSON arrays- "json_array": "_json",- "jsonb_array": "_jsonb",-- // Network arrays- "inet_array": "_inet",- "cidr_array": "_cidr",- "macaddr_array": "_macaddr",- "macaddr8_array": "_macaddr8",-- // Geometric arrays- "point_array": "_point",- "line_array": "_line",- "lseg_array": "_lseg",- "box_array": "_box",- "path_array": "_path",- "polygon_array": "_polygon",- "circle_array": "_circle",-- // Full-text search arrays- "tsvector_array": "_tsvector",- "tsquery_array": "_tsquery",-- // Range arrays- "int4range_array": "_int4range",- "int8range_array": "_int8range",- "numrange_array": "_numrange",- "tsrange_array": "_tsrange",- "tstzrange_array": "_tstzrange",- "daterange_array": "_daterange",-- // Multi-range arrays (PostgreSQL 14+)- "int4multirange_array": "_int4multirange",- "int8multirange_array": "_int8multirange",- "nummultirange_array": "_nummultirange",- "tsmultirange_array": "_tsmultirange",- "tstzmultirange_array": "_tstzmultirange",- "datemultirange_array": "_datemultirange",-- // Bit string arrays- "bit_array": "_bit",- "bit_with_length": "_bit",- "bit_varying_array": "_varbit",- "varbit_array": "_varbit",- "varbit_with_length": "_varbit",-- // Object identifier arrays- "oid_array": "_oid",- "regclass_array": "_regclass",- "regproc_array": "_regproc",- "regprocedure_array": "_regprocedure",- "regoper_array": "_regoper",- "regoperator_array": "_regoperator",- "regtype_array": "_regtype",- "regconfig_array": "_regconfig",- "regdictionary_array": "_regdictionary",- "regnamespace_array": "_regnamespace",- "regrole_array": "_regrole",- "regcollation_array": "_regcollation",-- // System type arrays- "tid_array": "_tid",- "xid_array": "_xid",- "xid8_array": "_xid8",- "cid_array": "_cid",- "pg_lsn_array": "_pg_lsn",-- // Custom enum arrays (should use fallback logic)- "status_array": "_status_enum",- "priority_array": "_priority_level",-- // Other arrays- "xml_array": "_xml",- "jsonpath_array": "_jsonpath",- "txid_snapshot_array": "_txid_snapshot",- }-- // Find the array_types_test table in both metadata structures- var syncTable, parseTable *storepb.TableMetadata-- for _, schema := range syncMetadata.Schemas {- for _, table := range schema.Tables {- if table.Name == "array_types_test" {- syncTable = table- break- }- }- if syncTable != nil {- break- }- }-- for _, schema := range parseMetadata.Schemas {- for _, table := range schema.Tables {- if table.Name == "array_types_test" {- parseTable = table- break- }- }- if parseTable != nil {- break- }- }-- require.NotNil(t, syncTable, "array_types_test table not found in sync metadata")- require.NotNil(t, parseTable, "array_types_test table not found in parse metadata")-- // Build maps of column types for easy lookup- syncColumnTypes := make(map[string]string)- parseColumnTypes := make(map[string]string)-- for _, col := range syncTable.Columns {- syncColumnTypes[col.Name] = col.Type- }-- for _, col := range parseTable.Columns {- parseColumnTypes[col.Name] = col.Type- }-- t.Log("Array type validation results:")- t.Log("Column Name | Sync Type | Parse Type | Expected | Status")- t.Log(strings.Repeat("-", 120))-- allCorrect := true-- for columnName, expectedType := range expectedArrayTypes {- syncType, syncExists := syncColumnTypes[columnName]- parseType, parseExists := parseColumnTypes[columnName]-- if !syncExists {- t.Errorf("Column %s not found in sync metadata", columnName)- allCorrect = false- continue- }-- if !parseExists {- t.Errorf("Column %s not found in parse metadata", columnName)- allCorrect = false- continue- }-- // The key validation: parse metadata should match expected array type- parseCorrect := parseType == expectedType- syncParseMatch := syncType == parseType-- status := "✓"- if !parseCorrect {- status = "✗ Parse incorrect"- allCorrect = false- } else if !syncParseMatch {- status = "⚠ Sync/Parse mismatch"- // This might be acceptable since sync gets actual PostgreSQL type names- // while parse gets our normalized versions- }-- t.Logf("%-30s | %-24s | %-24s | %-12s | %s",- columnName, syncType, parseType, expectedType, status)-- // The critical assertion: our parsed metadata should have the correct array types- if !parseCorrect {- t.Errorf("Array type mismatch for column %s: got %s, expected %s", columnName, parseType, expectedType)- }- }-- if allCorrect {- t.Log("🎉 All array types are correctly mapped!")- } else {- t.Error("❌ Some array types are not correctly mapped")- }-}diff --git a/backend/plugin/schema/pg/omni_filter_test.go b/backend/plugin/schema/pg/omni_filter_test.gonew file mode 100644index 0000000000..e3118f2a50--- /dev/null+++ b/backend/plugin/schema/pg/omni_filter_test.go@@ -0,0 +1,144 @@+package pg++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/schema"+ "github.com/bytebase/bytebase/backend/store/model"+)++// TestOmniFilter_BbdataarchiveSchemaFiltered verifies that objects in the+// bbdataarchive schema are excluded from migration output.+func TestOmniFilter_BbdataarchiveSchemaFiltered(t *testing.T) {+ // Source has tables in both public and bbdataarchive schemas.+ sourceMetadata := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {Name: "users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ {+ Name: "bbdataarchive",+ Tables: []*storepb.TableMetadata{+ {Name: "archived_users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ },+ }+ // Target is empty — should only generate DROP for public, not bbdataarchive.+ targetSDL := ""++ sourceMeta := model.NewDatabaseMetadata(sourceMetadata, nil, nil, storepb.Engine_POSTGRES, false)+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, sourceMeta)+ require.NoError(t, err)++ sql, err := schema.DiffSDLMigration(storepb.Engine_POSTGRES, sourceSDL, targetSDL)+ require.NoError(t, err)++ // bbdataarchive objects must not appear in migration.+ require.NotContains(t, sql, "bbdataarchive")+ require.NotContains(t, sql, "archived_users")+ // public objects should be dropped.+ require.Contains(t, sql, "users")+}++// TestOmniFilter_SkipBackupSchemaInSDLGeneration verifies that MetadataToSDL+// excludes the backup schema entirely from generated SDL text.+func TestOmniFilter_SkipBackupSchemaInSDLGeneration(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {Name: "t1", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ {+ Name: "bbdataarchive",+ Tables: []*storepb.TableMetadata{+ {Name: "backup_t1", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ },+ }+ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)++ sdl, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)+ require.NoError(t, err)++ require.NotContains(t, sdl, "bbdataarchive")+ require.NotContains(t, sdl, "backup_t1")+ require.Contains(t, sdl, "t1")+}++// TestOmniFilter_SkipDumpObjectsExcluded verifies that objects marked with+// SkipDump (e.g., extension-created objects) are excluded from SDL generation.+func TestOmniFilter_SkipDumpObjectsExcluded(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {Name: "user_table", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ {Name: "ext_table", SkipDump: true, Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ Functions: []*storepb.FunctionMetadata{+ {Name: "my_func", Definition: "CREATE FUNCTION my_func() RETURNS void LANGUAGE sql AS $$ SELECT 1 $$;"},+ {Name: "ext_func", SkipDump: true, Definition: "CREATE FUNCTION ext_func() RETURNS void LANGUAGE sql AS $$ SELECT 1 $$;"},+ },+ },+ },+ }+ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)++ sdl, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)+ require.NoError(t, err)++ require.Contains(t, sdl, "user_table")+ require.Contains(t, sdl, "my_func")+ require.NotContains(t, sdl, "ext_table")+ require.NotContains(t, sdl, "ext_func")+}++// TestOmniFilter_ExtensionPreservedWhenArchiveFiltered verifies that extension+// operations are preserved even when bbdataarchive objects are filtered.+func TestOmniFilter_ExtensionPreservedWhenArchiveFiltered(t *testing.T) {+ sql := omniSDLMigration(t, "", `CREATE EXTENSION IF NOT EXISTS "citext";`)+ // Extension creation should not be filtered.+ require.Contains(t, sql, "citext")+}++// TestOmniFilter_NoChangesForIdenticalSchemas verifies that two identical+// schemas with backup objects produce no migration (backup objects are+// excluded from both sides consistently).+func TestOmniFilter_NoChangesForIdenticalSchemas(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {Name: "users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ {+ Name: "bbdataarchive",+ Tables: []*storepb.TableMetadata{+ {Name: "old_users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},+ },+ },+ },+ }++ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)+ // Use MetadataToSDL to get the exact SDL that would be generated, then diff against itself.+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)+ require.NoError(t, err)+ sql, err := schema.SDLMigration(storepb.Engine_POSTGRES, sourceSDL, meta)+ require.NoError(t, err)+ require.Empty(t, sql, "identical public schemas with backup objects should produce no migration")+}diff --git a/backend/plugin/schema/pg/omni_parsing_migration_test.go b/backend/plugin/schema/pg/omni_parsing_migration_test.gonew file mode 100644index 0000000000..81faeef033--- /dev/null+++ b/backend/plugin/schema/pg/omni_parsing_migration_test.go@@ -0,0 +1,102 @@+package pg++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/omni/pg/catalog"+)++// Group 1: ALTER parsing tests (migrated from pg_alter_parsing_test.go)++func TestOmniAlterColumnIdentity(t *testing.T) {+ sql := omniSDLMigration(t, "", `CREATE TABLE products (id integer GENERATED BY DEFAULT AS IDENTITY NOT NULL);`)+ require.Contains(t, sql, "CREATE TABLE")+ require.Contains(t, sql, "products")+}++func TestOmniIdentityEquivalence(t *testing.T) {+ sdl := `CREATE TABLE orders (id integer GENERATED BY DEFAULT AS IDENTITY NOT NULL, amount numeric);`+ sql := omniSDLMigration(t, sdl, sdl)+ require.Empty(t, sql)+}++func TestOmniSequenceOwnedBy(t *testing.T) {+ // Verify that a standalone sequence (without OWNED BY) can be loaded and diffed.+ // The OWNED BY clause requires the referenced table to already exist in the catalog,+ // so we test sequence creation separately and table+sequence together without OWNED BY.+ sql := omniSDLMigration(t, "", `CREATE TABLE users (id integer); CREATE SEQUENCE user_seq;`)+ require.NotEmpty(t, sql)+ require.Contains(t, sql, "user_seq")+}++// Group 2: Extension sync bug tests (migrated from extension_test.go)++func TestOmniExtensionWithVersion(t *testing.T) {+ sdl := `CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public" VERSION '1.6';`+ _, err := catalog.LoadSDL(strings.TrimSpace(sdl))+ require.NoError(t, err)+}++func TestOmniExtensionCommentNoPhantomSchema(t *testing.T) {+ // Verify that loading an extension SDL does not create a phantom schema named after the extension.+ toSDL := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";`+ to, err := catalog.LoadSDL(strings.TrimSpace(toSDL))+ require.NoError(t, err)+ require.NotNil(t, to)++ // Adding a real table alongside the extension should not include a "pg_trgm" schema.+ toSDL2 := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";+CREATE TABLE test_table (id integer PRIMARY KEY);`+ sql := omniSDLMigration(t, "", toSDL2)+ require.NotContains(t, sql, `CREATE SCHEMA "pg_trgm"`)+ require.NotContains(t, sql, "CREATE SCHEMA pg_trgm")+}++func TestOmniExtensionNoDiffWhenIdentical(t *testing.T) {+ sdl := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";`+ sql := omniSDLMigration(t, sdl, sdl)+ require.Empty(t, sql)+}++// Group 3: SDL validation tests (migrated from sdl_validation_testcontainer_test.go)++func TestOmniSDLValidation_CreateFromEmpty(t *testing.T) {+ sql := omniSDLMigration(t, "", `+ CREATE TABLE users (id serial PRIMARY KEY, name varchar(255) NOT NULL, email varchar(255) UNIQUE);+ CREATE TABLE orders (id serial PRIMARY KEY, user_id integer REFERENCES users(id), total decimal(10,2));+ CREATE INDEX idx_orders_user_id ON orders(user_id);+ `)+ require.Contains(t, sql, "users")+ require.Contains(t, sql, "orders")+ require.Contains(t, sql, "idx_orders_user_id")+}++func TestOmniSDLValidation_AddViewAndFunction(t *testing.T) {+ fromSDL := `CREATE TABLE users (id serial PRIMARY KEY, name text NOT NULL, active boolean DEFAULT true);`+ toSDL := fromSDL + `+ CREATE VIEW active_users AS SELECT id, name FROM users WHERE active = true;+ CREATE FUNCTION count_active() RETURNS integer LANGUAGE sql AS $$ SELECT count(*) FROM users WHERE active = true $$;+ `+ sql := omniSDLMigration(t, fromSDL, toSDL)+ require.Contains(t, sql, "VIEW")+ require.Contains(t, sql, "FUNCTION")+}++func TestOmniSDLValidation_DropAllObjects(t *testing.T) {+ fromSDL := `CREATE TABLE t1 (id serial PRIMARY KEY); CREATE TABLE t2 (id serial PRIMARY KEY, ref integer REFERENCES t1(id));`+ sql := omniSDLMigration(t, fromSDL, "")+ require.Contains(t, sql, "DROP")+}++func TestOmniSDLValidation_CrossSchemaReferences(t *testing.T) {+ sql := omniSDLMigration(t, "", `+ CREATE SCHEMA app;+ CREATE TABLE app.users (id serial PRIMARY KEY, name text NOT NULL);+ CREATE TABLE app.orders (id serial PRIMARY KEY, user_id integer REFERENCES app.users(id));+ `)+ require.Contains(t, sql, "CREATE SCHEMA")+ require.Contains(t, sql, "CREATE TABLE")+}diff --git a/backend/plugin/schema/pg/pg_alter_parsing_test.go b/backend/plugin/schema/pg/pg_alter_parsing_test.godeleted file mode 100644index 7eee92c051..0000000000--- a/backend/plugin/schema/pg/pg_alter_parsing_test.go+++ /dev/null@@ -1,154 +0,0 @@-package pg--import (- "testing"-- "github.com/stretchr/testify/require"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"- "github.com/bytebase/bytebase/backend/plugin/schema"- "github.com/bytebase/bytebase/backend/store/model"-)--// TestAlterColumnAddIdentityParsing tests that ALTER COLUMN ADD IDENTITY is correctly parsed-func TestAlterColumnAddIdentityParsing(t *testing.T) {- ddl := `ALTER TABLE orders ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY;`-- parseResults, err := pgparser.ParsePostgreSQL(ddl)- require.NoError(t, err)- require.Len(t, parseResults, 1, "Should parse single statement")- require.NotNil(t, parseResults[0].Tree, "Parse tree should exist")-}--// TestAlterColumnAddIdentityMetadata tests ALTER COLUMN ADD IDENTITY creates sequences-func TestAlterColumnAddIdentityMetadata(t *testing.T) {- ddl := `- CREATE TABLE products (- id integer NOT NULL- );-- ALTER TABLE products ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (- SEQUENCE NAME products_id_seq- START WITH 1- INCREMENT BY 1- );- `-- metadata, err := GetDatabaseMetadata(ddl)- require.NoError(t, err)- require.NotNil(t, metadata)-- require.NotEmpty(t, metadata.Schemas, "Should have schemas")- schema := metadata.Schemas[0]-- require.Len(t, schema.Tables, 1, "Should have one table")- table := schema.Tables[0]- require.Equal(t, "products", table.Name)-- require.Len(t, table.Columns, 1, "Should have one column")- column := table.Columns[0]- require.Equal(t, "id", column.Name)- require.Equal(t, storepb.ColumnMetadata_BY_DEFAULT, column.IdentityGeneration)-- require.Len(t, schema.Sequences, 1, "Should have one sequence")- seq := schema.Sequences[0]- require.Equal(t, "products_id_seq", seq.Name)- require.Equal(t, "products", seq.OwnerTable)- require.Equal(t, "id", seq.OwnerColumn)-}--// TestAlterColumnIdentityEquivalence tests that two equivalent schemas don't generate diffs-func TestAlterColumnIdentityEquivalence(t *testing.T) {- // User pasted schema (ALTER COLUMN ADD IDENTITY format)- userSchema := `- CREATE TABLE "public"."orders" (- "id" integer NOT NULL,- "amount" numeric- );-- ALTER TABLE "public"."orders" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY (- SEQUENCE NAME "public"."orders_id_seq"- START WITH 1- INCREMENT BY 1- MINVALUE 1- MAXVALUE 2147483647- CACHE 1- );- `-- // Database current state (inline IDENTITY format)- dbSchema := `- CREATE TABLE orders (- id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,- amount NUMERIC- );- `-- userMetadata, err := GetDatabaseMetadata(userSchema)- require.NoError(t, err)- require.NotNil(t, userMetadata)-- dbMetadata, err := GetDatabaseMetadata(dbSchema)- require.NoError(t, err)- require.NotNil(t, dbMetadata)-- // Simulate database sync adding LastValue- for _, seq := range dbMetadata.Schemas[0].Sequences {- seq.LastValue = "1"- }-- // Compare schemas- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)- require.NoError(t, err)-- // Key assertion: no sequence changes should be detected- require.Empty(t, diff.SequenceChanges, "Should not detect sequence changes for equivalent schemas")-}--// TestAlterSequenceOwnedBy tests ALTER SEQUENCE OWNED BY parsing-func TestAlterSequenceOwnedBy(t *testing.T) {- tests := []struct {- name string- schema string- expectOwnerTable string- expectOwnerColumn string- }{- {- name: "ALTER SEQUENCE with OWNED BY",- schema: `- CREATE SEQUENCE user_seq;- CREATE TABLE users (id INT);- ALTER SEQUENCE user_seq OWNED BY users.id;- `,- expectOwnerTable: "users",- expectOwnerColumn: "id",- },- {- name: "CREATE SEQUENCE without OWNED BY",- schema: `- CREATE SEQUENCE user_seq;- `,- expectOwnerTable: "",- expectOwnerColumn: "",- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- metadata, err := GetDatabaseMetadata(tt.schema)- require.NoError(t, err)- require.NotNil(t, metadata)-- require.NotEmpty(t, metadata.Schemas)- require.NotEmpty(t, metadata.Schemas[0].Sequences)-- seq := metadata.Schemas[0].Sequences[0]- require.Equal(t, tt.expectOwnerTable, seq.OwnerTable, "OwnerTable mismatch")- require.Equal(t, tt.expectOwnerColumn, seq.OwnerColumn, "OwnerColumn mismatch")- })- }-}diff --git a/backend/plugin/schema/pg/pg_comment_parsing_test.go b/backend/plugin/schema/pg/pg_comment_parsing_test.godeleted file mode 100644index ff50afb3aa..0000000000--- a/backend/plugin/schema/pg/pg_comment_parsing_test.go+++ /dev/null@@ -1,301 +0,0 @@-package pg--import (- "testing"-- "github.com/stretchr/testify/require"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/schema"- "github.com/bytebase/bytebase/backend/store/model"-)--// TestCommentOnTable tests COMMENT ON TABLE parsing-func TestCommentOnTable(t *testing.T) {- // User pasted schema- userSchema := `- CREATE TABLE "public"."invoices" (- "id" integer NOT NULL,- "total" numeric- );-- COMMENT ON TABLE "public"."invoices" IS 'Invoice records';- `-- // Database current state- dbSchema := `- CREATE TABLE invoices (- id INT NOT NULL,- total NUMERIC- );-- COMMENT ON TABLE invoices IS 'Invoice records';- `-- userMetadata, err := GetDatabaseMetadata(userSchema)- require.NoError(t, err)- require.NotNil(t, userMetadata)-- dbMetadata, err := GetDatabaseMetadata(dbSchema)- require.NoError(t, err)- require.NotNil(t, dbMetadata)-- // Verify table comments are parsed- require.NotEmpty(t, userMetadata.Schemas)- require.NotEmpty(t, userMetadata.Schemas[0].Tables)- require.Equal(t, "Invoice records", userMetadata.Schemas[0].Tables[0].Comment)-- require.NotEmpty(t, dbMetadata.Schemas)- require.NotEmpty(t, dbMetadata.Schemas[0].Tables)- require.Equal(t, "Invoice records", dbMetadata.Schemas[0].Tables[0].Comment)-- // Compare schemas- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)- require.NoError(t, err)-- migration, err := generateMigration(diff)- require.NoError(t, err)-- // Should not generate COMMENT statements when comments are identical- require.NotContains(t, migration, "COMMENT ON TABLE", "Should not generate COMMENT statements for unchanged comments")-}--// TestCommentOnColumn tests COMMENT ON COLUMN parsing including Chinese characters-func TestCommentOnColumn(t *testing.T) {- // User pasted schema with column comments- userSchema := `- CREATE TABLE "public"."orders" (- "id" integer NOT NULL,- "old_plan_id" text,- "new_plan_id" text- );-- ALTER TABLE "public"."orders" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY (- SEQUENCE NAME "public"."orders_id_seq"- START WITH 1- INCREMENT BY 1- );-- COMMENT ON COLUMN "public"."orders"."old_plan_id" IS 'Previous plan identifier';- COMMENT ON COLUMN "public"."orders"."new_plan_id" IS 'Current plan identifier';- `-- // Database current state with same comments- dbSchema := `- CREATE TABLE orders (- id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,- old_plan_id TEXT,- new_plan_id TEXT- );-- COMMENT ON COLUMN orders.old_plan_id IS 'Previous plan identifier';- COMMENT ON COLUMN orders.new_plan_id IS 'Current plan identifier';- `-- userMetadata, err := GetDatabaseMetadata(userSchema)- require.NoError(t, err)-- dbMetadata, err := GetDatabaseMetadata(dbSchema)- require.NoError(t, err)-- // Verify column comments are parsed- userTable := userMetadata.Schemas[0].Tables[0]- require.Len(t, userTable.Columns, 3)- require.Equal(t, "", userTable.Columns[0].Comment)- require.Equal(t, "Previous plan identifier", userTable.Columns[1].Comment)- require.Equal(t, "Current plan identifier", userTable.Columns[2].Comment)-- dbTable := dbMetadata.Schemas[0].Tables[0]- require.Len(t, dbTable.Columns, 3)- require.Equal(t, "", dbTable.Columns[0].Comment)- require.Equal(t, "Previous plan identifier", dbTable.Columns[1].Comment)- require.Equal(t, "Current plan identifier", dbTable.Columns[2].Comment)-- // Compare schemas- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)- require.NoError(t, err)-- migration, err := generateMigration(diff)- require.NoError(t, err)-- // Should not generate COMMENT statements when comments are identical- require.NotContains(t, migration, "COMMENT ON COLUMN", "Should not generate COMMENT statements for unchanged comments")-}--// TestMultilineComment tests multiline comment parsing-func TestMultilineComment(t *testing.T) {- userSchema := `- CREATE TABLE "public"."transactions" (- "source_type" text- );-- COMMENT ON COLUMN "public"."transactions"."source_type" IS 'Transaction source types:-0: Mobile app-1: Web portal-2: API integration-3: Partner system-4: Admin panel';- `-- dbSchema := `- CREATE TABLE transactions (- source_type TEXT- );-- COMMENT ON COLUMN transactions.source_type IS 'Transaction source types:-0: Mobile app-1: Web portal-2: API integration-3: Partner system-4: Admin panel';- `-- userMetadata, err := GetDatabaseMetadata(userSchema)- require.NoError(t, err)-- dbMetadata, err := GetDatabaseMetadata(dbSchema)- require.NoError(t, err)-- // Verify multiline comments are preserved- userCol := userMetadata.Schemas[0].Tables[0].Columns[0]- dbCol := dbMetadata.Schemas[0].Tables[0].Columns[0]-- require.Contains(t, userCol.Comment, "\n", "Should preserve newlines")- require.Equal(t, userCol.Comment, dbCol.Comment, "Comments should be identical")-- // Compare schemas- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)- require.NoError(t, err)-- migration, err := generateMigration(diff)- require.NoError(t, err)-- // Should not generate COMMENT statements- require.Empty(t, migration, "Should not generate migration for identical comments")-}--// TestAllCommentTypes tests all supported comment types-func TestAllCommentTypes(t *testing.T) {- schema := `- CREATE SCHEMA app;- COMMENT ON SCHEMA app IS 'Application schema';-- CREATE TABLE app.users (- id INT,- username TEXT- );- COMMENT ON TABLE app.users IS 'User accounts';- COMMENT ON COLUMN app.users.id IS 'User identifier';- COMMENT ON COLUMN app.users.username IS 'Login username';-- CREATE INDEX idx_users_username ON app.users(username);- COMMENT ON INDEX app.idx_users_username IS 'Index for username lookups';-- CREATE VIEW app.active_users AS SELECT * FROM app.users;- COMMENT ON VIEW app.active_users IS 'Currently active users';-- CREATE SEQUENCE app.user_id_seq;- COMMENT ON SEQUENCE app.user_id_seq IS 'User ID sequence generator';-- CREATE TYPE app.user_status AS ENUM ('active', 'inactive', 'suspended');- COMMENT ON TYPE app.user_status IS 'User account status';-- CREATE FUNCTION app.get_user(user_id INT) RETURNS TEXT AS $$- BEGIN- RETURN 'user';- END;- $$ LANGUAGE plpgsql;- COMMENT ON FUNCTION app.get_user(INT) IS 'Retrieve user information';- `-- metadata, err := GetDatabaseMetadata(schema)- require.NoError(t, err)- require.NotNil(t, metadata)-- // Find the app schema- var appSchema *storepb.SchemaMetadata- for _, s := range metadata.Schemas {- if s.Name == "app" {- appSchema = s- break- }- }- require.NotNil(t, appSchema, "Should find app schema")-- // Verify all comment types- require.Equal(t, "Application schema", appSchema.Comment)- require.NotEmpty(t, appSchema.Tables)- require.Equal(t, "User accounts", appSchema.Tables[0].Comment)- require.NotEmpty(t, appSchema.Tables[0].Columns)- require.Equal(t, "User identifier", appSchema.Tables[0].Columns[0].Comment)- require.NotEmpty(t, appSchema.Tables[0].Indexes)- require.Equal(t, "Index for username lookups", appSchema.Tables[0].Indexes[0].Comment)- require.NotEmpty(t, appSchema.Views)- require.Equal(t, "Currently active users", appSchema.Views[0].Comment)- require.NotEmpty(t, appSchema.Sequences)- require.Equal(t, "User ID sequence generator", appSchema.Sequences[0].Comment)- require.NotEmpty(t, appSchema.EnumTypes)- require.Equal(t, "User account status", appSchema.EnumTypes[0].Comment)- require.NotEmpty(t, appSchema.Functions)- require.Equal(t, "Retrieve user information", appSchema.Functions[0].Comment)-}--// TestCommentEquivalence tests that identical comments don't generate migrations-func TestCommentEquivalence(t *testing.T) {- sourceSchema := `- CREATE TABLE products (- id INT,- name TEXT- );- COMMENT ON TABLE products IS 'Product catalog';- COMMENT ON COLUMN products.id IS 'Product ID';-- CREATE INDEX idx_name ON products(name);- COMMENT ON INDEX idx_name IS 'Product name index';-- CREATE SEQUENCE product_seq;- COMMENT ON SEQUENCE product_seq IS 'Product sequence';- `-- targetSchema := `- CREATE TABLE products (- id INT,- name TEXT- );- COMMENT ON TABLE products IS 'Product catalog';- COMMENT ON COLUMN products.id IS 'Product ID';-- CREATE INDEX idx_name ON products(name);- COMMENT ON INDEX idx_name IS 'Product name index';-- CREATE SEQUENCE product_seq;- COMMENT ON SEQUENCE product_seq IS 'Product sequence';- `-- sourceMetadata, err := GetDatabaseMetadata(sourceSchema)- require.NoError(t, err)-- targetMetadata, err := GetDatabaseMetadata(targetSchema)- require.NoError(t, err)-- sourceDBSchema := model.NewDatabaseMetadata(sourceMetadata, nil, nil, storepb.Engine_POSTGRES, false)- targetDBSchema := model.NewDatabaseMetadata(targetMetadata, nil, nil, storepb.Engine_POSTGRES, false)-- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, sourceDBSchema, targetDBSchema)- require.NoError(t, err)-- migration, err := generateMigration(diff)- require.NoError(t, err)-- // Key assertion: no COMMENT statements should be generated- require.NotContains(t, migration, "COMMENT ON", "Should not generate COMMENT statements for identical comments")-}diff --git a/backend/plugin/schema/pg/sdl_migration_omni.go b/backend/plugin/schema/pg/sdl_migration_omni.goindex c547fa2bf0..3156aa3a32 100644--- a/backend/plugin/schema/pg/sdl_migration_omni.go+++ b/backend/plugin/schema/pg/sdl_migration_omni.go@@ -15,114 +15,75 @@ import ()func init() {- schema.RegisterSDLMigration(storepb.Engine_POSTGRES, pgSDLMigration)- schema.RegisterSDLMigration(storepb.Engine_COCKROACHDB, pgSDLMigration)+ schema.RegisterDiffSDLMigration(storepb.Engine_POSTGRES, pgDiffSDLMigration)+ schema.RegisterDiffSDLMigration(storepb.Engine_COCKROACHDB, pgDiffSDLMigration)schema.RegisterSDLDropAdvices(storepb.Engine_POSTGRES, pgSDLDropAdvices)schema.RegisterSDLDropAdvices(storepb.Engine_COCKROACHDB, pgSDLDropAdvices)- schema.RegisterDiffMigration(storepb.Engine_POSTGRES, pgSchemaDiffMigration)- schema.RegisterDiffMigration(storepb.Engine_COCKROACHDB, pgSchemaDiffMigration)}-// catalogFromMetadata builds an omni Catalog from database metadata.-func catalogFromMetadata(meta *model.DatabaseMetadata) (*catalog.Catalog, error) {- if meta == nil {+// loadCatalog loads a schema text into a catalog. It tries LoadSDL first+// (declarative CREATE/COMMENT/GRANT only). If that fails (e.g. raw dump with+// SET/SELECT), it falls back to LoadSQL which accepts any SQL.+func loadCatalog(text string) (*catalog.Catalog, error) {+ text = strings.TrimSpace(text)+ if text == "" {return catalog.LoadSDL("")}- proto := meta.GetProto()- if proto == nil {- return catalog.LoadSDL("")- }- ddl, err := getSDLFormat(proto)+ c, err := catalog.LoadSDL(text)if err != nil {- return nil, errors.Wrap(err, "failed to convert metadata to DDL")+ return catalog.LoadSQL(text)}- return catalog.LoadSDL(ddl)+ return c, nil}-// pgSchemaDiffMigration computes migration SQL between two metadata states using omni.-func pgSchemaDiffMigration(oldSchema, newSchema *model.DatabaseMetadata) (string, error) {- from, err := catalogFromMetadata(oldSchema)+// buildMigrationPlan loads two schema texts into catalogs, diffs them, and+// returns the filtered migration plan. Returns nil if there are no changes.+func buildMigrationPlan(sourceText, targetText string) (*catalog.MigrationPlan, error) {+ from, err := loadCatalog(sourceText)if err != nil {- return "", errors.Wrap(err, "failed to load source schema")+ return nil, errors.Wrap(err, "failed to load source schema")}- to, err := catalogFromMetadata(newSchema)+ to, err := loadCatalog(targetText)if err != nil {- return "", errors.Wrap(err, "failed to load target schema")+ return nil, errors.Wrap(err, "failed to load target schema")}diff := catalog.Diff(from, to)if diff.IsEmpty() {- return "", nil- }- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))- return plan.SQL(), nil-}--// buildSDLCatalogs builds the from/to catalogs for an SDL migration.-func buildSDLCatalogs(userSDLText string, currentSchema *model.DatabaseMetadata) (*catalog.Catalog, *catalog.Catalog, error) {- var fromDDL string- if currentSchema != nil {- proto := currentSchema.GetProto()- if proto != nil {- var ddlErr error- fromDDL, ddlErr = getSDLFormat(proto)- if ddlErr != nil {- return nil, nil, errors.Wrap(ddlErr, "failed to convert current schema to DDL")- }- }- }-- from, err := catalog.LoadSDL(fromDDL)- if err != nil {- return nil, nil, errors.Wrapf(err, "failed to load current schema into catalog")- }-- to, err := catalog.LoadSDL(strings.TrimSpace(userSDLText))- if err != nil {- return nil, nil, errors.Wrapf(err, "failed to load user SDL into catalog")+ return nil, nil}-- return from, to, nil-}--// filterArchiveOps filters out bbdataarchive schema changes from a migration plan.-func filterArchiveOps(plan *catalog.MigrationPlan) *catalog.MigrationPlan {- return plan.Filter(func(op catalog.MigrationOp) bool {+ plan := catalog.GenerateMigration(from, to, diff)+ plan = plan.Filter(func(op catalog.MigrationOp) bool {return op.SchemaName != "bbdataarchive"})+ return plan, nil}-// pgSDLMigration computes the migration SQL from a user-provided SDL text and-// the current database schema using the omni catalog engine.-func pgSDLMigration(userSDLText string, currentSchema *model.DatabaseMetadata) (string, error) {- from, to, err := buildSDLCatalogs(userSDLText, currentSchema)+// pgDiffSDLMigration is the core migration function: two schema texts in, migration SQL out.+func pgDiffSDLMigration(sourceSDL, targetSDL string) (string, error) {+ plan, err := buildMigrationPlan(sourceSDL, targetSDL)if err != nil {return "", err}-- diff := catalog.Diff(from, to)- if diff.IsEmpty() {+ if plan == nil {return "", nil}-- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))return plan.SQL(), nil}-// pgSDLDropAdvices analyzes the SDL migration plan for destructive operations-// and returns warnings.+// pgSDLDropAdvices analyzes the SDL migration plan for destructive operations.func pgSDLDropAdvices(userSDLText string, currentSchema *model.DatabaseMetadata) ([]*storepb.Advice, error) {- from, to, err := buildSDLCatalogs(userSDLText, currentSchema)+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, currentSchema)if err != nil {return nil, err}-- diff := catalog.Diff(from, to)- if diff.IsEmpty() {+ plan, err := buildMigrationPlan(sourceSDL, userSDLText)+ if err != nil {+ return nil, err+ }+ if plan == nil {return nil, nil}- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))-var advices []*storepb.Advicefor _, op := range plan.Ops {switch op.Type {diff --git a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.godeleted file mode 100644index 89ead04f61..0000000000--- a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go+++ /dev/null@@ -1,1633 +0,0 @@-package pg--import (- "context"- "encoding/json"- "fmt"- "os"- "path/filepath"- "strings"- "testing"- "time"-- "github.com/pkg/errors"- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/schema"- "github.com/bytebase/bytebase/backend/store/model"-)--// TestSDLValidationWithTestcontainer tests the SDL (Schema Definition Language) correctness-// by following a 5-step validation process:-// 1. Apply initial schema A to database-// 2. Sync to get schema B from database-// 3. Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL-// 4. Apply generated DDL to database, then sync to get schema D-// 5. Verify schema D and schema C metadata have no diff and validate each member consistently-//-//nolint:tparallel-func TestSDLValidationWithTestcontainer(t *testing.T) {- ctx := context.Background()-- // Start shared PostgreSQL container for all subtests- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- // Test cases following the 5-step SDL validation process- testCases := []struct {- name string- initialSchema string // Schema A - initial state- expectedSchema string // Schema C - target state- description string- }{- {- name: "hr_test",- initialSchema: `--SET statement_timeout = 0;-SET lock_timeout = 0;-SET idle_in_transaction_session_timeout = 0;-SET client_encoding = 'UTF8';-SET standard_conforming_strings = on;-SELECT pg_catalog.set_config('search_path', '', false);-SET check_function_bodies = false;-SET xmloption = content;-SET client_min_messages = warning;-SET row_security = off;--SET default_tablespace = '';--CREATE SEQUENCE "public"."audit_id_seq"- AS integer- START WITH 1- INCREMENT BY 1- MINVALUE 1- MAXVALUE 2147483647- NO CYCLE;--CREATE TABLE "public"."audit" (- "id" integer DEFAULT nextval('public.audit_id_seq'::regclass) NOT NULL,- "operation" text NOT NULL,- "query" text,- "user_name" text NOT NULL,- "changed_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP-);--ALTER SEQUENCE "public"."audit_id_seq" OWNED BY "public"."audit"."id";--ALTER TABLE ONLY "public"."audit" ADD CONSTRAINT "audit_pkey" PRIMARY KEY (id);--CREATE INDEX "idx_audit_changed_at" ON "public"."audit" (changed_at);--CREATE INDEX "idx_audit_operation" ON "public"."audit" (operation);--CREATE INDEX "idx_audit_username" ON "public"."audit" (user_name);--CREATE TABLE "public"."department" (- "dept_no" text NOT NULL,- "dept_name" text NOT NULL-);--ALTER TABLE ONLY "public"."department" ADD CONSTRAINT "department_pkey" PRIMARY KEY (dept_no);--ALTER TABLE ONLY "public"."department" ADD CONSTRAINT "department_dept_name_key" UNIQUE ("dept_name");--CREATE TABLE "public"."dept_emp" (- "emp_no" integer NOT NULL,- "dept_no" text NOT NULL,- "from_date" date NOT NULL,- "to_date" date NOT NULL-);--ALTER TABLE ONLY "public"."dept_emp" ADD CONSTRAINT "dept_emp_pkey" PRIMARY KEY (emp_no, dept_no);--CREATE TABLE "public"."dept_manager" (- "emp_no" integer NOT NULL,- "dept_no" text NOT NULL,- "from_date" date NOT NULL,- "to_date" date NOT NULL-);--ALTER TABLE ONLY "public"."dept_manager" ADD CONSTRAINT "dept_manager_pkey" PRIMARY KEY (emp_no, dept_no);--CREATE SEQUENCE "public"."employee_emp_no_seq"- AS integer- START WITH 1- INCREMENT BY 1- MINVALUE 1- MAXVALUE 2147483647- NO CYCLE;--CREATE TABLE "public"."employee" (- "emp_no" integer DEFAULT nextval('public.employee_emp_no_seq'::regclass) NOT NULL,- "birth_date" date NOT NULL,- "first_name" text NOT NULL,- "last_name" text NOT NULL,- "gender" text NOT NULL,- "hire_date" date NOT NULL,- CONSTRAINT "employee_gender_check" CHECK (gender = ANY (ARRAY['M'::text, 'F'::text]))-);--ALTER SEQUENCE "public"."employee_emp_no_seq" OWNED BY "public"."employee"."emp_no";--ALTER TABLE ONLY "public"."employee" ADD CONSTRAINT "employee_pkey" PRIMARY KEY (emp_no);--CREATE INDEX "idx_employee_hire_date" ON "public"."employee" (hire_date);--CREATE OR REPLACE FUNCTION public.log_dml_operations()- RETURNS trigger- LANGUAGE plpgsql-AS $function$-BEGIN- IF (TG_OP = 'INSERT') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('INSERT', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'UPDATE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('UPDATE', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'DELETE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('DELETE', current_query(), current_user);- RETURN OLD;- END IF;- RETURN NULL;-END;-$function$-;--CREATE TABLE "public"."salary" (- "emp_no" integer NOT NULL,- "amount" integer NOT NULL,- "from_date" date NOT NULL,- "to_date" date NOT NULL-);--ALTER TABLE ONLY "public"."salary" ADD CONSTRAINT "salary_pkey" PRIMARY KEY (emp_no, from_date);--CREATE INDEX "idx_salary_amount" ON "public"."salary" (amount);--CREATE TABLE "public"."title" (- "emp_no" integer NOT NULL,- "title" text NOT NULL,- "from_date" date NOT NULL,- "to_date" date-);--ALTER TABLE ONLY "public"."title" ADD CONSTRAINT "title_pkey" PRIMARY KEY (emp_no, title, from_date);--CREATE VIEW "public"."dept_emp_latest_date" AS- SELECT emp_no,- max(from_date) AS from_date,- max(to_date) AS to_date- FROM public.dept_emp- GROUP BY emp_no;--CREATE VIEW "public"."current_dept_emp" AS- SELECT l.emp_no,- d.dept_no,- l.from_date,- l.to_date- FROM (public.dept_emp d- JOIN public.dept_emp_latest_date l ON (((d.emp_no = l.emp_no) AND (d.from_date = l.from_date) AND (l.to_date = d.to_date))));--CREATE TRIGGER salary_log_trigger AFTER DELETE OR UPDATE ON public.salary FOR EACH ROW EXECUTE FUNCTION public.log_dml_operations();--ALTER TABLE "public"."dept_emp"- ADD CONSTRAINT "dept_emp_dept_no_fkey" FOREIGN KEY ("dept_no")- REFERENCES "public"."department" ("dept_no")- ON DELETE CASCADE;--ALTER TABLE "public"."dept_emp"- ADD CONSTRAINT "dept_emp_emp_no_fkey" FOREIGN KEY ("emp_no")- REFERENCES "public"."employee" ("emp_no")- ON DELETE CASCADE;--ALTER TABLE "public"."dept_manager"- ADD CONSTRAINT "dept_manager_dept_no_fkey" FOREIGN KEY ("dept_no")- REFERENCES "public"."department" ("dept_no")- ON DELETE CASCADE;--ALTER TABLE "public"."dept_manager"- ADD CONSTRAINT "dept_manager_emp_no_fkey" FOREIGN KEY ("emp_no")- REFERENCES "public"."employee" ("emp_no")- ON DELETE CASCADE;--ALTER TABLE "public"."salary"- ADD CONSTRAINT "salary_emp_no_fkey" FOREIGN KEY ("emp_no")- REFERENCES "public"."employee" ("emp_no")- ON DELETE CASCADE;--ALTER TABLE "public"."title"- ADD CONSTRAINT "title_emp_no_fkey" FOREIGN KEY ("emp_no")- REFERENCES "public"."employee" ("emp_no")- ON DELETE CASCADE;-`,- expectedSchema: `- CREATE TABLE public.employee (- emp_no SERIAL NOT NULL,- birth_date DATE NOT NULL,- first_name TEXT NOT NULL,- last_name TEXT NOT NULL,- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,- hire_date DATE NOT NULL,- PRIMARY KEY (emp_no)-);--CREATE INDEX idx_employee_hire_date ON public.employee (hire_date);--CREATE TABLE public.department (- dept_no TEXT NOT NULL,- dept_name TEXT NOT NULL,- PRIMARY KEY (dept_no),- UNIQUE (dept_name)-);--CREATE TABLE public.dept_manager (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE public.dept_emp (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE public.title (- emp_no INT NOT NULL,- title TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, title, from_date)-);--CREATE TABLE public.salary (- emp_no INT NOT NULL,- amount INT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, from_date)-);--CREATE INDEX idx_salary_amount ON public.salary (amount);--CREATE TABLE public.audit (- id SERIAL PRIMARY KEY,- operation TEXT NOT NULL,- query TEXT,- user_name TEXT NOT NULL,- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_audit_operation ON public.audit (operation);-CREATE INDEX idx_audit_username ON public.audit (user_name);-CREATE INDEX idx_audit_changed_at ON public.audit (changed_at);--CREATE OR REPLACE FUNCTION public.log_dml_operations() RETURNS TRIGGER AS $$-BEGIN- IF (TG_OP = 'INSERT') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('INSERT', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'UPDATE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('UPDATE', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'DELETE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('DELETE', current_query(), current_user);- RETURN OLD;- END IF;- RETURN NULL;-END;-$$ LANGUAGE plpgsql;---- only log update and delete, otherwise, it will cause too much change.-CREATE TRIGGER salary_log_trigger-AFTER UPDATE OR DELETE ON public.salary-FOR EACH ROW-EXECUTE FUNCTION public.log_dml_operations();--CREATE OR REPLACE VIEW public.dept_emp_latest_date AS-SELECT- emp_no,- MAX(- from_date) AS from_date,- MAX(- to_date) AS to_date-FROM- public.dept_emp-GROUP BY- emp_no;---- shows only the current department for each employee-CREATE OR REPLACE VIEW public.current_dept_emp AS-SELECT- l.emp_no,- dept_no,- l.from_date,- l.to_date-FROM- public.dept_emp d- INNER JOIN public.dept_emp_latest_date l ON d.emp_no = l.emp_no- AND d.from_date = l.from_date- AND l.to_date = d.to_date;--CREATE TABLE users (- id SERIAL PRIMARY KEY,- username TEXT NOT NULL,- email TEXT NOT NULL,- password TEXT NOT NULL-);-`,- },- {- name: "start_for_empty_schema",- initialSchema: ``,- expectedSchema: `- CREATE TABLE public.employee (- emp_no SERIAL NOT NULL,- birth_date DATE NOT NULL,- first_name TEXT NOT NULL,- last_name TEXT NOT NULL,- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,- hire_date DATE NOT NULL,- PRIMARY KEY (emp_no)-);--CREATE INDEX idx_employee_hire_date ON public.employee (hire_date);--CREATE TABLE public.department (- dept_no TEXT NOT NULL,- dept_name TEXT NOT NULL,- PRIMARY KEY (dept_no),- UNIQUE (dept_name)-);--CREATE TABLE public.dept_manager (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE public.dept_emp (- emp_no INT NOT NULL,- dept_no TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, dept_no)-);--CREATE TABLE public.title (- emp_no INT NOT NULL,- title TEXT NOT NULL,- from_date DATE NOT NULL,- to_date DATE,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, title, from_date)-);--CREATE TABLE public.salary (- emp_no INT NOT NULL,- amount INT NOT NULL,- from_date DATE NOT NULL,- to_date DATE NOT NULL,- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,- PRIMARY KEY (emp_no, from_date)-);--CREATE INDEX idx_salary_amount ON public.salary (amount);--CREATE TABLE public.audit (- id SERIAL PRIMARY KEY,- operation TEXT NOT NULL,- query TEXT,- user_name TEXT NOT NULL,- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_audit_operation ON public.audit (operation);-CREATE INDEX idx_audit_username ON public.audit (user_name);-CREATE INDEX idx_audit_changed_at ON public.audit (changed_at);--CREATE OR REPLACE FUNCTION public.log_dml_operations() RETURNS TRIGGER AS $$-BEGIN- IF (TG_OP = 'INSERT') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('INSERT', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'UPDATE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('UPDATE', current_query(), current_user);- RETURN NEW;- ELSIF (TG_OP = 'DELETE') THEN- INSERT INTO public.audit (operation, query, user_name)- VALUES ('DELETE', current_query(), current_user);- RETURN OLD;- END IF;- RETURN NULL;-END;-$$ LANGUAGE plpgsql;---- only log update and delete, otherwise, it will cause too much change.-CREATE TRIGGER salary_log_trigger-AFTER UPDATE OR DELETE ON public.salary-FOR EACH ROW-EXECUTE FUNCTION public.log_dml_operations();--CREATE OR REPLACE VIEW public.dept_emp_latest_date AS-SELECT- emp_no,- MAX(- from_date) AS from_date,- MAX(- to_date) AS to_date-FROM- public.dept_emp-GROUP BY- emp_no;---- shows only the current department for each employee-CREATE OR REPLACE VIEW public.current_dept_emp AS-SELECT- l.emp_no,- dept_no,- l.from_date,- l.to_date-FROM- public.dept_emp d- INNER JOIN public.dept_emp_latest_date l ON d.emp_no = l.emp_no- AND d.from_date = l.from_date- AND l.to_date = d.to_date;- `,- description: "Start with an empty schema",- },- {- name: "add_table_with_constraints",- initialSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);-`,- expectedSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE TABLE posts (- id SERIAL PRIMARY KEY,- user_id INTEGER NOT NULL,- title VARCHAR(200) NOT NULL,- content TEXT,- published BOOLEAN DEFAULT false,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE-);--CREATE INDEX idx_posts_user_id ON posts(user_id);-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;-`,- description: "Add new table with foreign key and indexes",- },- {- name: "modify_table_add_columns",- initialSchema: `-CREATE TABLE products (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- price DECIMAL(10, 2) NOT NULL-);-`,- expectedSchema: `-CREATE TABLE products (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- price DECIMAL(10, 2) NOT NULL,- description TEXT,- category VARCHAR(50),- is_active BOOLEAN DEFAULT true,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_products_category ON products(category);-CREATE INDEX idx_products_active ON products(is_active) WHERE is_active = true;-`,- description: "Add columns and indexes to existing table",- },- {- name: "add_view_and_function",- initialSchema: `-CREATE TABLE employees (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- department VARCHAR(50),- salary DECIMAL(10, 2),- hire_date DATE DEFAULT CURRENT_DATE-);-`,- expectedSchema: `-CREATE TABLE employees (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- department VARCHAR(50),- salary DECIMAL(10, 2),- hire_date DATE DEFAULT CURRENT_DATE-);--CREATE VIEW active_employees AS-SELECT id, name, department, salary-FROM employees-WHERE department IS NOT NULL;--CREATE FUNCTION get_employee_count(dept VARCHAR) RETURNS INTEGER AS $$-BEGIN- RETURN (SELECT COUNT(*) FROM employees WHERE department = dept);-END;-$$ LANGUAGE plpgsql;--CREATE FUNCTION calculate_bonus(emp_id INTEGER) RETURNS DECIMAL AS $$-DECLARE- emp_salary DECIMAL;-BEGIN- SELECT salary INTO emp_salary FROM employees WHERE id = emp_id;- RETURN COALESCE(emp_salary * 0.1, 0);-END;-$$ LANGUAGE plpgsql;-`,- description: "Add view and functions to existing table",- },- {- name: "add_enum_and_sequence",- initialSchema: `-CREATE TABLE basic_items (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL-);-`,- expectedSchema: `-CREATE TYPE status_enum AS ENUM ('pending', 'active', 'inactive', 'deleted');-CREATE TYPE mood AS ENUM ('happy', 'sad', 'neutral');--CREATE SEQUENCE custom_id_seq START WITH 1000 INCREMENT BY 10;--CREATE TABLE basic_items (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL-);--CREATE TABLE items (- id INTEGER DEFAULT nextval('custom_id_seq') PRIMARY KEY,- name VARCHAR(100) NOT NULL,- status status_enum DEFAULT 'pending',- user_mood mood,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE INDEX idx_items_status ON items(status);-`,- description: "Add enum types and custom sequence",- },- {- name: "add_triggers_and_audit",- initialSchema: `-CREATE TABLE sales (- id SERIAL PRIMARY KEY,- product_name VARCHAR(100) NOT NULL,- amount DECIMAL(10, 2) NOT NULL,- sale_date DATE DEFAULT CURRENT_DATE-);-`,- expectedSchema: `-CREATE TABLE sales (- id SERIAL PRIMARY KEY,- product_name VARCHAR(100) NOT NULL,- amount DECIMAL(10, 2) NOT NULL,- sale_date DATE DEFAULT CURRENT_DATE-);--CREATE TABLE audit_log (- id SERIAL PRIMARY KEY,- table_name VARCHAR(50) NOT NULL,- operation VARCHAR(10) NOT NULL,- user_id INTEGER,- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- old_values JSONB,- new_values JSONB-);--CREATE OR REPLACE FUNCTION audit_trigger_function()-RETURNS TRIGGER AS $$-BEGIN- INSERT INTO audit_log (table_name, operation, old_values, new_values)- VALUES (- TG_TABLE_NAME,- TG_OP,- CASE WHEN TG_OP = 'DELETE' THEN row_to_json(OLD) ELSE NULL END,- CASE WHEN TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN row_to_json(NEW) ELSE NULL END- );- RETURN COALESCE(NEW, OLD);-END;-$$ LANGUAGE plpgsql;--CREATE TRIGGER sales_audit_trigger- AFTER INSERT OR UPDATE OR DELETE ON sales- FOR EACH ROW- EXECUTE FUNCTION audit_trigger_function();-`,- description: "Add audit table, function and trigger",- },- {- name: "create_from_empty",- initialSchema: ``, // Start from empty database- expectedSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE TABLE posts (- id SERIAL PRIMARY KEY,- user_id INTEGER NOT NULL,- title VARCHAR(200) NOT NULL,- content TEXT,- published BOOLEAN DEFAULT false,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE-);--CREATE INDEX idx_posts_user_id ON posts(user_id);-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;--CREATE VIEW user_post_count AS-SELECT u.username, COUNT(p.id) as post_count-FROM users u-LEFT JOIN posts p ON u.id = p.user_id-GROUP BY u.id, u.username;-`,- description: "Create complete schema from empty database",- },- {- name: "cross_schema_references",- initialSchema: `-CREATE SCHEMA hr;--CREATE TABLE hr.departments (- id SERIAL PRIMARY KEY,- name VARCHAR(50) NOT NULL UNIQUE-);-`,- expectedSchema: `-CREATE SCHEMA hr;-CREATE SCHEMA finance;--CREATE TABLE hr.departments (- id SERIAL PRIMARY KEY,- name VARCHAR(50) NOT NULL UNIQUE,- manager_id INTEGER-);--CREATE TABLE hr.employees (- id SERIAL PRIMARY KEY,- first_name VARCHAR(50) NOT NULL,- last_name VARCHAR(50) NOT NULL,- department_id INTEGER NOT NULL,- salary DECIMAL(10, 2),- hire_date DATE DEFAULT CURRENT_DATE,- CONSTRAINT fk_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id)-);---- Self-referencing foreign key for manager-ALTER TABLE hr.departments-ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES hr.employees(id);---- Tables in finance schema-CREATE TABLE finance.budgets (- id SERIAL PRIMARY KEY,- department_id INTEGER NOT NULL,- fiscal_year INTEGER NOT NULL,- allocated_amount DECIMAL(12, 2) NOT NULL,- spent_amount DECIMAL(12, 2) DEFAULT 0.00,- -- Cross-schema foreign key- CONSTRAINT fk_budget_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id),- CONSTRAINT unique_budget_year UNIQUE (department_id, fiscal_year)-);---- View that joins across schemas-CREATE VIEW finance.department_spending AS-SELECT d.name AS department_name,- b.fiscal_year,- b.allocated_amount,- b.spent_amount,- (b.allocated_amount - b.spent_amount) AS remaining_budget-FROM hr.departments d- JOIN finance.budgets b ON d.id = b.department_id;-`,- description: "Add cross-schema references and views",- },- {- name: "drop_all_objects",- initialSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP-);--CREATE TABLE posts (- id SERIAL PRIMARY KEY,- user_id INTEGER NOT NULL,- title VARCHAR(200) NOT NULL,- content TEXT,- published BOOLEAN DEFAULT false,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE-);--CREATE INDEX idx_posts_user_id ON posts(user_id);-`,- expectedSchema: ``, // Empty schema - expect all objects to be dropped- description: "Drop all objects from database (test deletion)",- },- {- name: "foreign_key_on_delete_cascade",- initialSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL-);-`,- expectedSchema: `-CREATE TABLE users (- id SERIAL PRIMARY KEY,- username VARCHAR(50) NOT NULL UNIQUE,- email VARCHAR(100) NOT NULL-);--CREATE TABLE posts (- id SERIAL PRIMARY KEY,- user_id INTEGER NOT NULL,- title VARCHAR(200) NOT NULL,- content TEXT,- published BOOLEAN DEFAULT false,- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE-);--CREATE TABLE comments (- id SERIAL PRIMARY KEY,- post_id INTEGER NOT NULL,- user_id INTEGER NOT NULL,- content TEXT NOT NULL,- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,- CONSTRAINT fk_comment_post FOREIGN KEY (post_id) REFERENCES posts(id) ON DELETE CASCADE,- CONSTRAINT fk_comment_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL-);-`,- description: "Test foreign keys with ON DELETE CASCADE and SET NULL",- },- {- name: "foreign_key_on_update_actions",- initialSchema: `-CREATE TABLE categories (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL UNIQUE,- code VARCHAR(10) NOT NULL UNIQUE-);-`,- expectedSchema: `-CREATE TABLE categories (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL UNIQUE,- code VARCHAR(10) NOT NULL UNIQUE-);--CREATE TABLE products (- id SERIAL PRIMARY KEY,- category_id INTEGER NOT NULL,- category_code VARCHAR(10) NOT NULL,- name VARCHAR(200) NOT NULL,- price DECIMAL(10,2) NOT NULL,- CONSTRAINT fk_product_category_id FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE RESTRICT ON UPDATE CASCADE,- CONSTRAINT fk_product_category_code FOREIGN KEY (category_code) REFERENCES categories(code) ON DELETE CASCADE ON UPDATE SET NULL-);--CREATE TABLE product_reviews (- id SERIAL PRIMARY KEY,- product_id INTEGER NOT NULL,- rating INTEGER CHECK (rating >= 1 AND rating <= 5),- comment TEXT,- CONSTRAINT fk_review_product FOREIGN KEY (product_id) REFERENCES products(id) ON DELETE CASCADE ON UPDATE RESTRICT-);-`,- description: "Test foreign keys with various ON DELETE and ON UPDATE actions",- },- {- name: "foreign_key_no_action",- initialSchema: `-CREATE TABLE departments (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- budget DECIMAL(15,2) DEFAULT 0-);-`,- expectedSchema: `-CREATE TABLE departments (- id SERIAL PRIMARY KEY,- name VARCHAR(100) NOT NULL,- budget DECIMAL(15,2) DEFAULT 0-);--CREATE TABLE employees (- id SERIAL PRIMARY KEY,- department_id INTEGER NOT NULL,- name VARCHAR(100) NOT NULL,- salary DECIMAL(10,2) NOT NULL,- manager_id INTEGER,- CONSTRAINT fk_employee_department FOREIGN KEY (department_id) REFERENCES departments(id) ON DELETE NO ACTION ON UPDATE NO ACTION,- CONSTRAINT fk_employee_manager FOREIGN KEY (manager_id) REFERENCES employees(id)-);-`,- description: "Test foreign keys with NO ACTION (explicit and implicit default)",- },- }-- for _, tc := range testCases {- t.Run(tc.name, func(t *testing.T) {- t.Parallel()-- // Create a new database for each test case with unique name- dbName := fmt.Sprintf("sdl_test_%s_%d", strings.ReplaceAll(tc.name, "-", "_"), time.Now().UnixNano()%1000000)- _, err := container.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))- require.NoError(t, err)-- // Create driver for the test database- driver, err := createPgDriver(ctx, container.GetHost(), container.GetPort(), dbName)- require.NoError(t, err)- defer driver.Close(ctx)-- // Execute the 5-step SDL validation process- t.Logf("Starting SDL validation process for: %s", tc.description)-- // Step 1: Apply initial schema A to database- t.Log("Step 1: Applying initial schema A to database")- if strings.TrimSpace(tc.initialSchema) != "" {- _, err = driver.GetDB().ExecContext(ctx, tc.initialSchema)- require.NoError(t, err, "failed to apply initial schema")- }-- // Step 2: Sync to get schema B from database- t.Log("Step 2: Syncing to get schema B from database")- schemaB, err := driver.SyncDBSchema(ctx)- require.NoError(t, err, "failed to sync schema B")-- // Step 3: Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL- t.Log("Step 3: Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL")-- // Get metadata for expected schema C using GetDatabaseMetadata- // Note: Empty expectedSchema means we expect an empty database (all objects dropped)- schemaCMetadata, err := GetDatabaseMetadata(tc.expectedSchema)- require.NoError(t, err, "failed to get expected schema C metadata")-- // Generate migration DDL from schema B to schema C using the differ- migrationDDL, actualDiff, err := generateMigrationDDLFromMetadata(schemaB, schemaCMetadata)- require.NoError(t, err, "Failed to generate migration DDL from metadata")-- t.Logf("Generated migration DDL (%d characters):\n%s", len(migrationDDL), migrationDDL)-- // Save or validate the actual diff (B to C) and DDL files- testName := strings.ReplaceAll(t.Name(), "/", "_")- err = validateOrSaveTestFiles(t, actualDiff, testName, migrationDDL)- require.NoError(t, err, "test file validation/creation failed")-- // Step 4: Apply generated DDL to database, then sync to get schema D- t.Log("Step 4: Apply generated DDL to database, then sync to get schema D")- if strings.TrimSpace(migrationDDL) != "" {- _, err = driver.GetDB().ExecContext(ctx, migrationDDL)- require.NoError(t, err, "failed to apply migration DDL")- }-- // Sync to get schema D after migration- schemaD, err := driver.SyncDBSchema(ctx)- require.NoError(t, err, "failed to sync schema D after migration")-- // Step 5: Verify schema D and schema C metadata have no diff and validate each member consistently- t.Log("Step 5: Verify schema D and schema C metadata have no diff and validate each member consistently")-- // Compare schema D with schema C metadata to ensure they match- err = validateSchemaConsistency(schemaD, schemaCMetadata)- require.NoError(t, err, "schema validation failed - schema D does not match expected schema C")-- t.Logf("✓ SDL validation process completed successfully for: %s", tc.description)- })- }-}--// generateMigrationDDLFromMetadata generates migration DDL from schema B to schema C using schema.differ-func generateMigrationDDLFromMetadata(schemaB, schemaC *storepb.DatabaseSchemaMetadata) (string, *schema.MetadataDiff, error) {- // Create model.DatabaseMetadata objects for comparison- modelSchemaB := model.NewDatabaseMetadata(schemaB, nil, nil, storepb.Engine_POSTGRES, true)- modelSchemaC := model.NewDatabaseMetadata(schemaC, nil, nil, storepb.Engine_POSTGRES, true)-- // Use the schema differ to compute differences- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, modelSchemaB, modelSchemaC)- if err != nil {- return "", nil, errors.Wrapf(err, "failed to compute schema difference")- }-- // Convert the metadata diff to DDL using schema.GenerateMigration- // This generates the actual migration DDL from the computed differences- migrationDDL, err := schema.GenerateMigration(storepb.Engine_POSTGRES, diff)- if err != nil {- return "", nil, errors.Wrapf(err, "failed to generate migration DDL from schema diff")- }-- return migrationDDL, diff, nil-}--// validateSchemaConsistency compares two schema metadata objects to ensure they are equivalent-func validateSchemaConsistency(schemaD, schemaC *storepb.DatabaseSchemaMetadata) error {- // Create model.DatabaseMetadata objects for comparison- // PostgreSQL is case-sensitive for object names but case-insensitive for details- modelSchemaD := model.NewDatabaseMetadata(schemaD, nil, nil, storepb.Engine_POSTGRES, true)- modelSchemaC := model.NewDatabaseMetadata(schemaC, nil, nil, storepb.Engine_POSTGRES, true)-- // Use the schema differ to compare the two schemas- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, modelSchemaD, modelSchemaC)- if err != nil {- return errors.Wrapf(err, "failed to compute schema difference")- }-- // Note: diff files are saved earlier in the process with the actual B to C differences-- // Check if there are any differences- if diff != nil {- var diffDetails []string-- // Check for any schema changes- if len(diff.SchemaChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("SchemaChanges: %d", len(diff.SchemaChanges)))- }- if len(diff.TableChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("TableChanges: %d", len(diff.TableChanges)))- // Add detailed table changes for debugging- for i, tableDiff := range diff.TableChanges {- diffDetails = append(diffDetails, fmt.Sprintf(" Table[%d]: %s table %s.%s (columns: %d, indexes: %d, checkConstraints: %d, foreignKeys: %d, triggers: %d)",- i, tableDiff.Action, tableDiff.SchemaName, tableDiff.TableName,- len(tableDiff.ColumnChanges), len(tableDiff.IndexChanges), len(tableDiff.CheckConstraintChanges), len(tableDiff.ForeignKeyChanges), len(tableDiff.TriggerChanges)))-- // Add column change details- for j, columnDiff := range tableDiff.ColumnChanges {- columnName := ""- if columnDiff.OldColumn != nil {- columnName = columnDiff.OldColumn.Name- } else if columnDiff.NewColumn != nil {- columnName = columnDiff.NewColumn.Name- }- diffDetails = append(diffDetails, fmt.Sprintf(" Column[%d]: %s column %s", j, columnDiff.Action, columnName))- }-- // Add index change details- for j, indexDiff := range tableDiff.IndexChanges {- indexName := ""- if indexDiff.OldIndex != nil {- indexName = indexDiff.OldIndex.Name- } else if indexDiff.NewIndex != nil {- indexName = indexDiff.NewIndex.Name- }- diffDetails = append(diffDetails, fmt.Sprintf(" Index[%d]: %s index %s", j, indexDiff.Action, indexName))-- // Show more details for debugging- if indexDiff.OldIndex != nil && indexDiff.NewIndex != nil {- oldExpr := ""- newExpr := ""- if len(indexDiff.OldIndex.Expressions) > 0 {- oldExpr = strings.Join(indexDiff.OldIndex.Expressions, ", ")- }- if len(indexDiff.NewIndex.Expressions) > 0 {- newExpr = strings.Join(indexDiff.NewIndex.Expressions, ", ")- }- diffDetails = append(diffDetails, fmt.Sprintf(" Old: expr=%s type=%s unique=%v primary=%v", oldExpr, indexDiff.OldIndex.Type, indexDiff.OldIndex.Unique, indexDiff.OldIndex.Primary))- diffDetails = append(diffDetails, fmt.Sprintf(" Old def: %s", indexDiff.OldIndex.Definition))- diffDetails = append(diffDetails, fmt.Sprintf(" New: expr=%s type=%s unique=%v primary=%v", newExpr, indexDiff.NewIndex.Type, indexDiff.NewIndex.Unique, indexDiff.NewIndex.Primary))- diffDetails = append(diffDetails, fmt.Sprintf(" New def: %s", indexDiff.NewIndex.Definition))- } else if indexDiff.OldIndex != nil {- oldExpr := ""- if len(indexDiff.OldIndex.Expressions) > 0 {- oldExpr = strings.Join(indexDiff.OldIndex.Expressions, ", ")- }- diffDetails = append(diffDetails, fmt.Sprintf(" Old: expr=%s type=%s unique=%v primary=%v", oldExpr, indexDiff.OldIndex.Type, indexDiff.OldIndex.Unique, indexDiff.OldIndex.Primary))- diffDetails = append(diffDetails, fmt.Sprintf(" Old def: %s", indexDiff.OldIndex.Definition))- } else if indexDiff.NewIndex != nil {- newExpr := ""- if len(indexDiff.NewIndex.Expressions) > 0 {- newExpr = strings.Join(indexDiff.NewIndex.Expressions, ", ")- }- diffDetails = append(diffDetails, fmt.Sprintf(" New: expr=%s type=%s unique=%v primary=%v", newExpr, indexDiff.NewIndex.Type, indexDiff.NewIndex.Unique, indexDiff.NewIndex.Primary))- diffDetails = append(diffDetails, fmt.Sprintf(" New def: %s", indexDiff.NewIndex.Definition))- }- }- }- }- if len(diff.ViewChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("ViewChanges: %d", len(diff.ViewChanges)))- for _, viewDiff := range diff.ViewChanges {- diffDetails = append(diffDetails, fmt.Sprintf(" View: %s.%s, Action: %v", viewDiff.SchemaName, viewDiff.ViewName, viewDiff.Action))- if viewDiff.OldView != nil && viewDiff.NewView != nil {- diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))- diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))- } else if viewDiff.OldView != nil {- diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))- } else if viewDiff.NewView != nil {- diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))- }- }- }- if len(diff.MaterializedViewChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("MaterializedViewChanges: %d", len(diff.MaterializedViewChanges)))- }- if len(diff.FunctionChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("FunctionChanges: %d", len(diff.FunctionChanges)))- }- if len(diff.ProcedureChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("ProcedureChanges: %d", len(diff.ProcedureChanges)))- }- if len(diff.SequenceChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("SequenceChanges: %d", len(diff.SequenceChanges)))- }- if len(diff.EnumTypeChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("EnumTypeChanges: %d", len(diff.EnumTypeChanges)))- }- if len(diff.EventChanges) > 0 {- diffDetails = append(diffDetails, fmt.Sprintf("EventChanges: %d", len(diff.EventChanges)))- }-- if len(diffDetails) > 0 {- return errors.Errorf("schema D does not match expected schema C: found differences - %s", strings.Join(diffDetails, ", "))- }- }-- return nil-}--// DiffSummary represents the structure for JSON serialization of schema differences-type DiffSummary struct {- HasDifferences bool `json:"hasDifferences"`- Summary string `json:"summary"`- TableChanges int `json:"tableChanges"`- ViewChanges int `json:"viewChanges"`- FunctionChanges int `json:"functionChanges"`- SchemaChanges int `json:"schemaChanges"`- SequenceChanges int `json:"sequenceChanges"`- EnumTypeChanges int `json:"enumTypeChanges"`- Details *DiffDetails `json:"details,omitempty"`-}--// DiffDetails provides detailed information about the differences-type DiffDetails struct {- Tables []TableChangeSummary `json:"tables,omitempty"`- Views []ViewChangeSummary `json:"views,omitempty"`- Functions []FunctionChangeSummary `json:"functions,omitempty"`- Schemas []SchemaChangeSummary `json:"schemas,omitempty"`-}--// TableChangeSummary summarizes table changes with detailed information-type TableChangeSummary struct {- Action string `json:"action"`- SchemaName string `json:"schemaName"`- TableName string `json:"tableName"`- ColumnChanges int `json:"columnChanges"`- IndexChanges int `json:"indexChanges"`- ConstraintChanges int `json:"constraintChanges"`- ForeignKeyChanges int `json:"foreignKeyChanges"`- // Detailed changes- Columns []ColumnChangeDetail `json:"columns,omitempty"`- Indexes []IndexChangeDetail `json:"indexes,omitempty"`- Constraints []ConstraintChangeDetail `json:"constraints,omitempty"`- ForeignKeys []ForeignKeyChangeDetail `json:"foreignKeys,omitempty"`-}--// ViewChangeSummary summarizes view changes-type ViewChangeSummary struct {- Action string `json:"action"`- SchemaName string `json:"schemaName"`- ViewName string `json:"viewName"`-}--// FunctionChangeSummary summarizes function changes-type FunctionChangeSummary struct {- Action string `json:"action"`- SchemaName string `json:"schemaName"`- FunctionName string `json:"functionName"`-}--// SchemaChangeSummary summarizes schema changes-type SchemaChangeSummary struct {- Action string `json:"action"`- SchemaName string `json:"schemaName"`-}--// ColumnChangeDetail provides detailed information about column changes-type ColumnChangeDetail struct {- Action string `json:"action"`- Name string `json:"name"`- OldColumn *ColumnInfo `json:"oldColumn,omitempty"`- NewColumn *ColumnInfo `json:"newColumn,omitempty"`-}--// ColumnInfo represents column metadata-type ColumnInfo struct {- Name string `json:"name"`- Type string `json:"type"`- Nullable bool `json:"nullable"`- DefaultValue string `json:"defaultValue,omitempty"`- Comment string `json:"comment,omitempty"`-}--// IndexChangeDetail provides detailed information about index changes-type IndexChangeDetail struct {- Action string `json:"action"`- Name string `json:"name"`- OldIndex *IndexInfo `json:"oldIndex,omitempty"`- NewIndex *IndexInfo `json:"newIndex,omitempty"`-}--// IndexInfo represents index metadata-type IndexInfo struct {- Name string `json:"name"`- Type string `json:"type,omitempty"`- Unique bool `json:"unique"`- Expressions []string `json:"expressions"`- KeyLength []int64 `json:"keyLength,omitempty"`- Descending []bool `json:"descending,omitempty"`-}--// ConstraintChangeDetail provides detailed information about constraint changes-type ConstraintChangeDetail struct {- Action string `json:"action"`- Name string `json:"name"`- OldConstraint *ConstraintInfo `json:"oldConstraint,omitempty"`- NewConstraint *ConstraintInfo `json:"newConstraint,omitempty"`-}--// ConstraintInfo represents constraint metadata-type ConstraintInfo struct {- Name string `json:"name"`- Type string `json:"type"`- Expression string `json:"expression,omitempty"`-}--// ForeignKeyChangeDetail provides detailed information about foreign key changes-type ForeignKeyChangeDetail struct {- Action string `json:"action"`- Name string `json:"name"`- OldForeignKey *ForeignKeyInfo `json:"oldForeignKey,omitempty"`- NewForeignKey *ForeignKeyInfo `json:"newForeignKey,omitempty"`-}--// ForeignKeyInfo represents foreign key metadata-type ForeignKeyInfo struct {- Name string `json:"name"`- Columns []string `json:"columns"`- ReferencedSchema string `json:"referencedSchema"`- ReferencedTable string `json:"referencedTable"`- ReferencedColumns []string `json:"referencedColumns"`- OnUpdate string `json:"onUpdate,omitempty"`- OnDelete string `json:"onDelete,omitempty"`- MatchType string `json:"matchType,omitempty"`-}--// createDiffSummary creates a structured summary from MetadataDiff-func createDiffSummary(diff *schema.MetadataDiff) *DiffSummary {- if diff == nil {- return &DiffSummary{- HasDifferences: false,- Summary: "No schema differences found - schemas match perfectly!",- }- }-- // Check if there are actually any differences- hasDifferences := len(diff.TableChanges) > 0 || len(diff.ViewChanges) > 0 ||- len(diff.FunctionChanges) > 0 || len(diff.SchemaChanges) > 0 ||- len(diff.SequenceChanges) > 0 || len(diff.EnumTypeChanges) > 0-- summary := &DiffSummary{- HasDifferences: hasDifferences,- TableChanges: len(diff.TableChanges),- ViewChanges: len(diff.ViewChanges),- FunctionChanges: len(diff.FunctionChanges),- SchemaChanges: len(diff.SchemaChanges),- SequenceChanges: len(diff.SequenceChanges),- EnumTypeChanges: len(diff.EnumTypeChanges),- }-- if !hasDifferences {- summary.Summary = "No schema differences found - schemas match perfectly!"- return summary- }-- // Create summary text- var summaryParts []string- if summary.TableChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("Table changes: %d", summary.TableChanges))- }- if summary.ViewChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("View changes: %d", summary.ViewChanges))- }- if summary.FunctionChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("Function changes: %d", summary.FunctionChanges))- }- if summary.SchemaChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("Schema changes: %d", summary.SchemaChanges))- }- if summary.SequenceChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("Sequence changes: %d", summary.SequenceChanges))- }- if summary.EnumTypeChanges > 0 {- summaryParts = append(summaryParts, fmt.Sprintf("Enum type changes: %d", summary.EnumTypeChanges))- }-- summary.Summary = strings.Join(summaryParts, ", ")-- // Add detailed information- details := &DiffDetails{}-- // Add table changes details- for _, tableChange := range diff.TableChanges {- tableSummary := TableChangeSummary{- Action: string(tableChange.Action),- SchemaName: tableChange.SchemaName,- TableName: tableChange.TableName,- ColumnChanges: len(tableChange.ColumnChanges),- IndexChanges: len(tableChange.IndexChanges),- ConstraintChanges: len(tableChange.CheckConstraintChanges),- ForeignKeyChanges: len(tableChange.ForeignKeyChanges),- }-- // Add detailed column changes- for _, colChange := range tableChange.ColumnChanges {- columnDetail := ColumnChangeDetail{- Action: string(colChange.Action),- Name: getColumnName(colChange),- }- if colChange.OldColumn != nil {- columnDetail.OldColumn = &ColumnInfo{- Name: colChange.OldColumn.Name,- Type: colChange.OldColumn.Type,- Nullable: colChange.OldColumn.Nullable,- DefaultValue: colChange.OldColumn.Default,- Comment: colChange.OldColumn.Comment,- }- }- if colChange.NewColumn != nil {- columnDetail.NewColumn = &ColumnInfo{- Name: colChange.NewColumn.Name,- Type: colChange.NewColumn.Type,- Nullable: colChange.NewColumn.Nullable,- DefaultValue: colChange.NewColumn.Default,- Comment: colChange.NewColumn.Comment,- }- }- tableSummary.Columns = append(tableSummary.Columns, columnDetail)- }-- // Add detailed index changes- for _, indexChange := range tableChange.IndexChanges {- indexDetail := IndexChangeDetail{- Action: string(indexChange.Action),- Name: getIndexName(indexChange),- }- if indexChange.OldIndex != nil {- indexDetail.OldIndex = &IndexInfo{- Name: indexChange.OldIndex.Name,- Type: indexChange.OldIndex.Type,- Unique: indexChange.OldIndex.Unique,- Expressions: indexChange.OldIndex.Expressions,- KeyLength: indexChange.OldIndex.KeyLength,- Descending: indexChange.OldIndex.Descending,- }- }- if indexChange.NewIndex != nil {- indexDetail.NewIndex = &IndexInfo{- Name: indexChange.NewIndex.Name,- Type: indexChange.NewIndex.Type,- Unique: indexChange.NewIndex.Unique,- Expressions: indexChange.NewIndex.Expressions,- KeyLength: indexChange.NewIndex.KeyLength,- Descending: indexChange.NewIndex.Descending,- }- }- tableSummary.Indexes = append(tableSummary.Indexes, indexDetail)- }-- // Add detailed constraint changes- for _, constraintChange := range tableChange.CheckConstraintChanges {- constraintDetail := ConstraintChangeDetail{- Action: string(constraintChange.Action),- Name: getConstraintName(constraintChange),- }- if constraintChange.OldCheckConstraint != nil {- constraintDetail.OldConstraint = &ConstraintInfo{- Name: constraintChange.OldCheckConstraint.Name,- Type: "CHECK",- Expression: constraintChange.OldCheckConstraint.Expression,- }- }- if constraintChange.NewCheckConstraint != nil {- constraintDetail.NewConstraint = &ConstraintInfo{- Name: constraintChange.NewCheckConstraint.Name,- Type: "CHECK",- Expression: constraintChange.NewCheckConstraint.Expression,- }- }- tableSummary.Constraints = append(tableSummary.Constraints, constraintDetail)- }-- // Add detailed foreign key changes- for _, fkChange := range tableChange.ForeignKeyChanges {- fkDetail := ForeignKeyChangeDetail{- Action: string(fkChange.Action),- Name: getForeignKeyName(fkChange),- }- if fkChange.OldForeignKey != nil {- fkDetail.OldForeignKey = &ForeignKeyInfo{- Name: fkChange.OldForeignKey.Name,- Columns: fkChange.OldForeignKey.Columns,- ReferencedSchema: fkChange.OldForeignKey.ReferencedSchema,- ReferencedTable: fkChange.OldForeignKey.ReferencedTable,- ReferencedColumns: fkChange.OldForeignKey.ReferencedColumns,- OnUpdate: fkChange.OldForeignKey.OnUpdate,- OnDelete: fkChange.OldForeignKey.OnDelete,- MatchType: fkChange.OldForeignKey.MatchType,- }- }- if fkChange.NewForeignKey != nil {- fkDetail.NewForeignKey = &ForeignKeyInfo{- Name: fkChange.NewForeignKey.Name,- Columns: fkChange.NewForeignKey.Columns,- ReferencedSchema: fkChange.NewForeignKey.ReferencedSchema,- ReferencedTable: fkChange.NewForeignKey.ReferencedTable,- ReferencedColumns: fkChange.NewForeignKey.ReferencedColumns,- OnUpdate: fkChange.NewForeignKey.OnUpdate,- OnDelete: fkChange.NewForeignKey.OnDelete,- MatchType: fkChange.NewForeignKey.MatchType,- }- }- tableSummary.ForeignKeys = append(tableSummary.ForeignKeys, fkDetail)- }-- details.Tables = append(details.Tables, tableSummary)- }-- // Add view changes details- for _, viewChange := range diff.ViewChanges {- details.Views = append(details.Views, ViewChangeSummary{- Action: string(viewChange.Action),- SchemaName: viewChange.SchemaName,- ViewName: viewChange.ViewName,- })- }-- // Add function changes details- for _, funcChange := range diff.FunctionChanges {- details.Functions = append(details.Functions, FunctionChangeSummary{- Action: string(funcChange.Action),- SchemaName: funcChange.SchemaName,- FunctionName: funcChange.FunctionName,- })- }-- // Add schema changes details- for _, schemaChange := range diff.SchemaChanges {- details.Schemas = append(details.Schemas, SchemaChangeSummary{- Action: string(schemaChange.Action),- SchemaName: schemaChange.SchemaName,- })- }-- if len(details.Tables) > 0 || len(details.Views) > 0 || len(details.Functions) > 0 || len(details.Schemas) > 0 {- summary.Details = details- }-- return summary-}--// Helper functions to extract names from diff structures-func getColumnName(colChange *schema.ColumnDiff) string {- if colChange.NewColumn != nil {- return colChange.NewColumn.Name- }- if colChange.OldColumn != nil {- return colChange.OldColumn.Name- }- return ""-}--func getIndexName(indexChange *schema.IndexDiff) string {- if indexChange.NewIndex != nil {- return indexChange.NewIndex.Name- }- if indexChange.OldIndex != nil {- return indexChange.OldIndex.Name- }- return ""-}--func getConstraintName(constraintChange *schema.CheckConstraintDiff) string {- if constraintChange.NewCheckConstraint != nil {- return constraintChange.NewCheckConstraint.Name- }- if constraintChange.OldCheckConstraint != nil {- return constraintChange.OldCheckConstraint.Name- }- return ""-}--func getForeignKeyName(fkChange *schema.ForeignKeyDiff) string {- if fkChange.NewForeignKey != nil {- return fkChange.NewForeignKey.Name- }- if fkChange.OldForeignKey != nil {- return fkChange.OldForeignKey.Name- }- return ""-}--// validateOrSaveTestFiles validates existing test files or creates new ones based on UPDATE_TEST_FILES environment variable-func validateOrSaveTestFiles(t *testing.T, diff *schema.MetadataDiff, testName string, ddlContent string) error {- // Check if we should update test files- updateFiles := os.Getenv("UPDATE_TEST_FILES") == "true"-- testDir := getTestDataDirectory(testName)- if testDir == "" {- // For hardcoded tests (TestSDLValidationWithTestcontainer), skip file validation- // Only TestSDLValidationFromTestData tests should have corresponding test data files- if strings.Contains(testName, "TestSDLValidationWithTestcontainer") {- t.Logf("Skipping file validation for hardcoded test: %s", testName)- return nil- }- return errors.Errorf("could not determine test data directory for test %s", testName)- }-- diffFilename := filepath.Join(testDir, "diff.json")- ddlFilename := filepath.Join(testDir, "ddl.sql")-- // Generate current content- currentDiffJSON, err := generateDiffJSON(diff)- if err != nil {- return errors.Wrapf(err, "failed to generate diff JSON")- }-- if updateFiles {- // Update mode: create/overwrite files- t.Logf("Updating test files in %s", testDir)-- // Save diff JSON file- if err := os.WriteFile(diffFilename, []byte(currentDiffJSON), 0644); err != nil {- return errors.Wrapf(err, "failed to write diff.json file")- }-- // Save DDL file- if err := os.WriteFile(ddlFilename, []byte(ddlContent), 0644); err != nil {- return errors.Wrapf(err, "failed to write ddl.sql file")- }-- t.Logf("Test files updated: %s (diff.json, ddl.sql)", testDir)- return nil- }-- // Validation mode: check if existing files match current content- t.Logf("Validating test files in %s", testDir)-- // Validate diff.json- if err := validateFileContent(diffFilename, currentDiffJSON, "diff.json"); err != nil {- return err- }-- // Validate ddl.sql- if err := validateFileContent(ddlFilename, ddlContent, "ddl.sql"); err != nil {- return err- }-- t.Logf("✓ Test files validated successfully: %s", testDir)- return nil-}--// generateDiffJSON generates JSON content from MetadataDiff-func generateDiffJSON(diff *schema.MetadataDiff) (string, error) {- diffSummary := createDiffSummary(diff)- jsonData, err := json.MarshalIndent(diffSummary, "", " ")- if err != nil {- return "", errors.Wrapf(err, "failed to marshal diff to JSON")- }- return string(jsonData), nil-}--// validateFileContent validates that the file content matches expected content-func validateFileContent(filename, expectedContent, fileType string) error {- if _, err := os.Stat(filename); os.IsNotExist(err) {- return errors.Errorf("%s file does not exist: %s. Run with UPDATE_TEST_FILES=true to create it", fileType, filename)- }-- actualContent, err := os.ReadFile(filename)- if err != nil {- return errors.Wrapf(err, "failed to read %s file: %s", fileType, filename)- }-- actualStr := strings.TrimSpace(string(actualContent))- expectedStr := strings.TrimSpace(expectedContent)-- if actualStr != expectedStr {- return errors.Errorf("%s content mismatch in %s.\nExpected:\n%s\n\nActual:\n%s\n\nRun with UPDATE_TEST_FILES=true to update the file",- fileType, filename, expectedStr, actualStr)- }-- return nil-}--// getTestDataDirectory converts a test name to the corresponding test data directory path-func getTestDataDirectory(testName string) string {- // Handle TestSDLValidationFromTestData_ prefix- var pathPart string- if strings.HasPrefix(testName, "TestSDLValidationFromTestData_") {- pathPart = strings.TrimPrefix(testName, "TestSDLValidationFromTestData_")- } else {- return ""- }-- // Convert from test name format to directory format- // TestSDLValidationFromTestData_data_types_basic_types_datetime_types -> data_types/basic_types/datetime_types-- // Based on the known test structure, parse the path correctly:- // data_types_basic_types_datetime_types- // schema_objects_views_complex_views- // constraints_indexes_foreign_keys_cascade_actions-- var testDataPath string- switch {- case strings.HasPrefix(pathPart, "data_types_basic_types_"):- // data_types_basic_types_datetime_types -> data_types/basic_types/datetime_types- suffix := strings.TrimPrefix(pathPart, "data_types_basic_types_")- testDataPath = filepath.Join("data_types", "basic_types", suffix)- case strings.HasPrefix(pathPart, "data_types_advanced_types_"):- // data_types_advanced_types_binary_json_uuid -> data_types/advanced_types/binary_json_uuid- suffix := strings.TrimPrefix(pathPart, "data_types_advanced_types_")- testDataPath = filepath.Join("data_types", "advanced_types", suffix)- case strings.HasPrefix(pathPart, "data_types_type_conversions_"):- // data_types_type_conversions_compatible_conversions -> data_types/type_conversions/compatible_conversions- suffix := strings.TrimPrefix(pathPart, "data_types_type_conversions_")- testDataPath = filepath.Join("data_types", "type_conversions", suffix)- case strings.HasPrefix(pathPart, "schema_objects_views_"):- // schema_objects_views_complex_views -> schema_objects/views/complex_views- suffix := strings.TrimPrefix(pathPart, "schema_objects_views_")- testDataPath = filepath.Join("schema_objects", "views", suffix)- case strings.HasPrefix(pathPart, "constraints_indexes_foreign_keys_"):- // constraints_indexes_foreign_keys_cascade_actions -> constraints_indexes/foreign_keys/cascade_actions- suffix := strings.TrimPrefix(pathPart, "constraints_indexes_foreign_keys_")- testDataPath = filepath.Join("constraints_indexes", "foreign_keys", suffix)- case strings.HasPrefix(pathPart, "table_operations_column_operations_"):- // table_operations_column_operations_add_simple_columns -> table_operations/column_operations/add_simple_columns- suffix := strings.TrimPrefix(pathPart, "table_operations_column_operations_")- testDataPath = filepath.Join("table_operations", "column_operations", suffix)- default:- // Fallback: try to split intelligently- parts := strings.Split(pathPart, "_")- if len(parts) >= 3 {- // Take first 2 parts as category, rest as test name- testDataPath = filepath.Join(strings.Join(parts[:2], "/"), strings.Join(parts[2:], "_"))- } else {- testDataPath = strings.ReplaceAll(pathPart, "_", "/")- }- }-- return filepath.Join("sdl_testdata", testDataPath)-}diff --git a/backend/plugin/schema/pg/walk_through_omni.go b/backend/plugin/schema/pg/walk_through_omni.goindex 8a3d46d5f9..fa864dbb5a 100644--- a/backend/plugin/schema/pg/walk_through_omni.go+++ b/backend/plugin/schema/pg/walk_through_omni.go@@ -1,14 +1,17 @@package pgimport (+ "context""fmt""slices"+ "strings""github.com/bytebase/omni/pg/catalog"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor/code""github.com/bytebase/bytebase/backend/plugin/parser/base"+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg""github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/store/model")@@ -466,3 +469,55 @@ func fkMatchToString(match byte) string {return "SIMPLE"}}++func extractViewDependencies(schemaMetadata *storepb.DatabaseSchemaMetadata) {+ for _, s := range schemaMetadata.Schemas {+ for _, view := range s.Views {+ view.DependencyColumns = getViewDependencies(view.Definition, s.Name, schemaMetadata)+ }+ for _, mv := range s.MaterializedViews {+ mv.DependencyColumns = getViewDependencies(mv.Definition, s.Name, schemaMetadata)+ }+ }+}++func getViewDependencies(viewDef string, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {+ queryStatement := strings.TrimSpace(viewDef)+ span, err := pgparser.GetQuerySpan(+ context.Background(),+ base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {+ dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, nil, nil, storepb.Engine_POSTGRES, false)+ return databaseName, dbMetadata, nil+ },+ ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {+ return []string{}, nil+ },+ },+ base.Statement{Text: queryStatement},+ "",+ schemaName,+ false,+ )+ if err != nil {+ return []*storepb.DependencyColumn{}+ }++ dependencyMap := make(map[string]*storepb.DependencyColumn)+ for sourceColumn := range span.SourceColumns {+ key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)+ if _, exists := dependencyMap[key]; !exists {+ dependencyMap[key] = &storepb.DependencyColumn{+ Schema: sourceColumn.Schema,+ Table: sourceColumn.Table,+ Column: "*",+ }+ }+ }++ var dependencies []*storepb.DependencyColumn+ for _, dep := range dependencyMap {+ dependencies = append(dependencies, dep)+ }+ return dependencies+}diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.goindex 74f711da98..5bf9a030a0 100644--- a/backend/plugin/schema/schema.go+++ b/backend/plugin/schema/schema.go@@ -24,9 +24,8 @@ var (getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)generateMigrations = make(map[storepb.Engine]generateMigration)getSDLDiffs = make(map[storepb.Engine]getSDLDiff)- sdlMigrations = make(map[storepb.Engine]sdlMigration)sdlDropAdvicesFns = make(map[storepb.Engine]sdlDropAdvices)- schemaDiffMigrations = make(map[storepb.Engine]schemaDiffMigration)+ diffSDLMigrations = make(map[storepb.Engine]diffSDLMigration)getMultiFileDatabaseDefinitions = make(map[storepb.Engine]getMultiFileDatabaseDefinition)walkThroughs = make(map[storepb.Engine]walkThrough)walkThroughsWithContext = make(map[storepb.Engine]walkThroughWithContext)@@ -44,9 +43,8 @@ type getSequenceDefinition func(string, *storepb.SequenceMetadata) (string, errotype getDatabaseMetadata func(string) (*storepb.DatabaseSchemaMetadata, error)type generateMigration func(*MetadataDiff) (string, error)type getSDLDiff func(currentSDLText, previousUserSDLText string, currentSchema *model.DatabaseMetadata) (*MetadataDiff, error)-type sdlMigration func(userSDLText string, currentSchema *model.DatabaseMetadata) (string, error)type sdlDropAdvices func(userSDLText string, currentSchema *model.DatabaseMetadata) ([]*storepb.Advice, error)-type schemaDiffMigration func(oldSchema, newSchema *model.DatabaseMetadata) (string, error)+type diffSDLMigration func(sourceSDL, targetSDL string) (string, error)type walkThrough func(*model.DatabaseMetadata, []base.AST) *storepb.Advicetype walkThroughWithContext func(WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice@@ -268,23 +266,15 @@ func GetSDLDiff(engine storepb.Engine, currentSDLText, previousUserSDLText strinreturn f(currentSDLText, previousUserSDLText, currentSchema)}-func RegisterSDLMigration(engine storepb.Engine, f sdlMigration) {- mux.Lock()- defer mux.Unlock()- if _, dup := sdlMigrations[engine]; dup {- panic(fmt.Sprintf("Register called twice %s", engine))- }- sdlMigrations[engine] = f-}-// SDLMigration computes the migration SQL from a user-provided SDL text and the-// current database schema. It combines diff and migration generation in one step.+// current database schema. It converts the current metadata to SDL, then diffs+// against the user SDL using DiffSDLMigration.func SDLMigration(engine storepb.Engine, userSDLText string, currentSchema *model.DatabaseMetadata) (string, error) {- f, ok := sdlMigrations[engine]- if !ok {- return "", errors.Errorf("engine %s is not supported for SDL migration", engine)+ sourceSDL, err := MetadataToSDL(engine, currentSchema)+ if err != nil {+ return "", errors.Wrap(err, "failed to convert current schema to SDL")}- return f(userSDLText, currentSchema)+ return DiffSDLMigration(engine, sourceSDL, userSDLText)}func RegisterSDLDropAdvices(engine storepb.Engine, f sdlDropAdvices) {@@ -305,21 +295,20 @@ func SDLDropAdvices(engine storepb.Engine, userSDLText string, currentSchema *moreturn f(userSDLText, currentSchema)}-func RegisterDiffMigration(engine storepb.Engine, f schemaDiffMigration) {- mux.Lock()- defer mux.Unlock()- if _, dup := schemaDiffMigrations[engine]; dup {- panic(fmt.Sprintf("Register called twice %s", engine))- }- schemaDiffMigrations[engine] = f-}-// DiffMigration computes the migration SQL between two database metadata states.-// Falls back to the legacy GetDatabaseSchemaDiff + GenerateMigration path if no-// engine-specific implementation is registered.+// For engines with DiffSDLMigration registered, converts both sides to SDL and diffs.+// Falls back to the legacy GetDatabaseSchemaDiff + GenerateMigration path otherwise.func DiffMigration(engine storepb.Engine, oldSchema, newSchema *model.DatabaseMetadata) (string, error) {- if f, ok := schemaDiffMigrations[engine]; ok {- return f(oldSchema, newSchema)+ if _, ok := diffSDLMigrations[engine]; ok {+ sourceSDL, err := MetadataToSDL(engine, oldSchema)+ if err != nil {+ return "", errors.Wrap(err, "failed to convert source schema to SDL")+ }+ targetSDL, err := MetadataToSDL(engine, newSchema)+ if err != nil {+ return "", errors.Wrap(err, "failed to convert target schema to SDL")+ }+ return DiffSDLMigration(engine, sourceSDL, targetSDL)}// Fallback to legacy path for engines that haven't migrated yet.diff, err := GetDatabaseSchemaDiff(engine, oldSchema, newSchema)@@ -329,6 +318,39 @@ func DiffMigration(engine storepb.Engine, oldSchema, newSchema *model.DatabaseMereturn GenerateMigration(engine, diff)}+// MetadataToSDL converts database metadata to SDL text using the registered GetDatabaseDefinition.+func MetadataToSDL(engine storepb.Engine, meta *model.DatabaseMetadata) (string, error) {+ if meta == nil {+ return "", nil+ }+ proto := meta.GetProto()+ if proto == nil {+ return "", nil+ }+ return GetDatabaseDefinition(engine, GetDefinitionContext{+ SkipBackupSchema: true,+ SDLFormat: true,+ }, proto)+}++func RegisterDiffSDLMigration(engine storepb.Engine, f diffSDLMigration) {+ mux.Lock()+ defer mux.Unlock()+ if _, dup := diffSDLMigrations[engine]; dup {+ panic(fmt.Sprintf("Register called twice %s", engine))+ }+ diffSDLMigrations[engine] = f+}++// DiffSDLMigration computes migration SQL between two SDL texts.+func DiffSDLMigration(engine storepb.Engine, sourceSDL, targetSDL string) (string, error) {+ f, ok := diffSDLMigrations[engine]+ if !ok {+ return "", errors.Errorf("engine %s is not supported for SDL diff migration", engine)+ }+ return f(sourceSDL, targetSDL)+}+func RegisterGetMultiFileDatabaseDefinition(engine storepb.Engine, f getMultiFileDatabaseDefinition) {mux.Lock()defer mux.Unlock()