Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(pg): delete GetDatabaseMetadata and unify migration to DiffSDLMigration [BYT-9010] (#19744)

rebelice8049b14fb776
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.go
index fc07ed083c..917ec191d1 100644
--- a/backend/api/v1/database_service.go
+++ b/backend/api/v1/database_service.go
@@ -706,30 +706,68 @@ func (s *DatabaseService) GetDatabaseSDLSchema(ctx context.Context, req *connect
// DiffSchema diff the database schema.
func (s *DatabaseService) DiffSchema(ctx context.Context, req *connect.Request[v1pb.DiffSchemaRequest]) (*connect.Response[v1pb.DiffSchemaResponse], error) {
- // Use unified SDL-based approach for all scenarios
+ engine, err := s.getParserEngine(ctx, req.Msg)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get parser engine"))
+ }
+
+ // PG/CockroachDB: use omni SDL diff path.
+ if engine == storepb.Engine_POSTGRES || engine == storepb.Engine_COCKROACHDB {
+ migrationSQL, err := s.diffSchemaViaSDL(ctx, req.Msg, engine)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))
+ }
+ return connect.NewResponse(&v1pb.DiffSchemaResponse{Diff: migrationSQL}), nil
+ }
+
+ // Other engines: legacy metadata diff path.
sourceDBSchema, err := s.getSourceDBMetadata(ctx, req.Msg)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get source schema"))
}
-
targetDBSchema, err := s.getTargetDBMetadata(ctx, req.Msg)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get target schema"))
}
-
- engine, err := s.getParserEngine(ctx, req.Msg)
+ migrationSQL, err := schema.DiffMigration(engine, sourceDBSchema, targetDBSchema)
if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get parser engine"))
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))
}
+ return connect.NewResponse(&v1pb.DiffSchemaResponse{Diff: migrationSQL}), nil
+}
- migrationSQL, err := schema.DiffMigration(engine, sourceDBSchema, targetDBSchema)
+// diffSchemaViaSDL handles PG/CockroachDB DiffSchema using omni SDL diff.
+func (s *DatabaseService) diffSchemaViaSDL(ctx context.Context, req *v1pb.DiffSchemaRequest, engine storepb.Engine) (string, error) {
+ sourceMetadata, err := s.getSourceDBMetadata(ctx, req)
if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to compute schema diff"))
+ return "", err
+ }
+ sourceSDL, err := schema.MetadataToSDL(engine, sourceMetadata)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to generate source SDL")
}
- return connect.NewResponse(&v1pb.DiffSchemaResponse{
- Diff: migrationSQL,
- }), nil
+ // Resolve target schema text.
+ var targetSDL string
+ switch {
+ case req.GetSchema() != "":
+ // Schema text from GetDatabaseSchema (raw dump) — passed directly.
+ // pgDiffSDLMigration handles LoadSDL/LoadSQL fallback internally.
+ targetSDL = req.GetSchema()
+ case req.GetChangelog() != "":
+ targetMetadata, err := s.getSourceDBMetadata(ctx, &v1pb.DiffSchemaRequest{Name: req.GetChangelog()})
+ if err != nil {
+ return "", errors.Wrap(err, "failed to resolve target changelog")
+ }
+ targetSDL, err = schema.MetadataToSDL(engine, targetMetadata)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to generate target SDL")
+ }
+ default:
+ return "", errors.Errorf("target must be either schema text or changelog")
+ }
+
+ return schema.DiffSDLMigration(engine, sourceSDL, targetSDL)
}
func (s *DatabaseService) getSourceDBMetadata(ctx context.Context, request *v1pb.DiffSchemaRequest) (*model.DatabaseMetadata, error) {
diff --git a/backend/plugin/schema/pg/camelcase_index_test.go b/backend/plugin/schema/pg/camelcase_index_test.go
index 6e9e9c51b3..a96f70717a 100644
--- a/backend/plugin/schema/pg/camelcase_index_test.go
+++ b/backend/plugin/schema/pg/camelcase_index_test.go
@@ -33,73 +33,6 @@ func TestQuoteIndexExpression(t *testing.T) {
}
// TestCamelCaseIndexColumnQuoting_SyncSchema reproduces https://github.com/bytebase/bytebase/issues/19348
-// via the Sync Schema path: the source schema is exported (with proper quoting), then
-// parsed into metadata by GetDatabaseMetadata (which calls generateIndexDefinition).
-// The parsed metadata's Definition must preserve quoting for CamelCase columns.
-func TestCamelCaseIndexColumnQuoting_SyncSchema(t *testing.T) {
- // This is the exported schema from the source database (pg_get_indexdef quotes correctly).
- sourceSchemaSQL := `
-CREATE TABLE "public"."ba_account" (
- "userId" text NOT NULL,
- "accountName" text
-);
-
-CREATE INDEX "ba_account_userid_idx" ON "public"."ba_account" ("userId");
-`
-
- // Parse the source schema into metadata (this is what getTargetDBMetadata does).
- parsedMetadata, err := GetDatabaseMetadata(sourceSchemaSQL)
- require.NoError(t, err)
-
- // Verify the parsed index Definition has quoted "userId".
- var indexDef string
- for _, s := range parsedMetadata.Schemas {
- for _, tbl := range s.Tables {
- for _, idx := range tbl.Indexes {
- if idx.Name == "ba_account_userid_idx" {
- indexDef = idx.Definition
- }
- }
- }
- }
- require.NotEmpty(t, indexDef, "index definition should be set")
- t.Logf("Parsed index Definition: %s", indexDef)
- require.Contains(t, indexDef, `"userId"`, "CamelCase column must be quoted in parsed Definition")
-
- // Now simulate the full Sync Schema diff: target has no index, source has it.
- oldMetadata := &storepb.DatabaseSchemaMetadata{
- Name: "testdb",
- Schemas: []*storepb.SchemaMetadata{
- {
- Name: "public",
- Tables: []*storepb.TableMetadata{
- {
- Name: "ba_account",
- Columns: []*storepb.ColumnMetadata{
- {Name: "userId", Type: "text"},
- {Name: "accountName", Type: "text"},
- },
- },
- },
- },
- },
- }
-
- oldSchema := model.NewDatabaseMetadata(oldMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- newSchema := model.NewDatabaseMetadata(parsedMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, oldSchema, newSchema)
- require.NoError(t, err)
- require.NotNil(t, diff)
-
- ddl, err := generateMigration(diff)
- require.NoError(t, err)
-
- t.Logf("Generated DDL:\n%s", ddl)
-
- require.Contains(t, ddl, `"userId"`, "CamelCase column in CREATE INDEX must be quoted")
-}
-
// TestCamelCaseConstraintQuoting_PrimaryKey verifies that PRIMARY KEY constraints
// on CamelCase columns are properly quoted in generated ALTER TABLE statements.
func TestCamelCaseConstraintQuoting_PrimaryKey(t *testing.T) {
diff --git a/backend/plugin/schema/pg/extension_test.go b/backend/plugin/schema/pg/extension_test.go
index 8b618edb2a..c3a007438e 100644
--- a/backend/plugin/schema/pg/extension_test.go
+++ b/backend/plugin/schema/pg/extension_test.go
@@ -319,118 +319,3 @@ func TestExtensionNotFilteredByArchiveSchemaFilter(t *testing.T) {
// 1. Incorrectly generates DROP EXTENSION even though the extension is identical in source and target
// 2. Generates CREATE SCHEMA for a phantom schema named after the extension
//
-// Root causes:
-// - Bug 1: EnterCreateextensionstmt doesn't parse VERSION from SQL → compareExtensions sees false mismatch
-// - Bug 2: EnterCommentstmt treats COMMENT ON EXTENSION as COMMENT ON SCHEMA → creates phantom schema
-func TestExtensionSchemaSyncBug(t *testing.T) {
- t.Run("Bug1_version_not_parsed_from_SQL", func(t *testing.T) {
- // The raw schema dump (produced by writeExtension) includes VERSION.
- // GetDatabaseMetadata must parse it back correctly.
- sql := `CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';`
-
- metadata, err := GetDatabaseMetadata(sql)
- require.NoError(t, err)
- require.NotNil(t, metadata)
- require.Len(t, metadata.Extensions, 1, "Should have exactly 1 extension")
-
- ext := metadata.Extensions[0]
- require.Equal(t, "pg_trgm", ext.Name)
- require.Equal(t, "tecbatch", ext.Schema)
- require.Equal(t, "1.6", ext.Version, "BUG: VERSION not parsed from CREATE EXTENSION statement")
- })
-
- t.Run("Bug2_comment_on_extension_creates_phantom_schema", func(t *testing.T) {
- // COMMENT ON EXTENSION should NOT create a schema entry.
- sql := `CREATE SCHEMA IF NOT EXISTS "tecbatch";
-CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';
-COMMENT ON EXTENSION "pg_trgm" IS 'text similarity measurement and index searching based on trigrams';`
-
- metadata, err := GetDatabaseMetadata(sql)
- require.NoError(t, err)
- require.NotNil(t, metadata)
-
- // Should only have "public" (always created) and "tecbatch" schemas.
- // Must NOT have a "pg_trgm" schema.
- for _, s := range metadata.Schemas {
- require.NotEqual(t, "pg_trgm", s.Name,
- "BUG: COMMENT ON EXTENSION created a phantom schema named after the extension")
- }
-
- // Extension description should be populated (not lost).
- require.Len(t, metadata.Extensions, 1)
- require.Equal(t, "text similarity measurement and index searching based on trigrams",
- metadata.Extensions[0].Description,
- "BUG: COMMENT ON EXTENSION description not stored in extension metadata")
- })
-
- t.Run("Full_pipeline_no_false_diff", func(t *testing.T) {
- // Simulate the exact Schema Sync flow:
- // "old" = target DB metadata from live DB sync (has version + description)
- // "new" = source DB metadata parsed from raw schema dump
-
- // --- "old" metadata: target DB (from getExtensions live query) ---
- oldMetadata := &storepb.DatabaseSchemaMetadata{
- Extensions: []*storepb.ExtensionMetadata{
- {
- Name: "pg_trgm",
- Schema: "tecbatch",
- Version: "1.6",
- Description: "text similarity measurement and index searching based on trigrams",
- },
- },
- Schemas: []*storepb.SchemaMetadata{
- {Name: "public"},
- {Name: "tecbatch"},
- },
- }
-
- // --- "new" SQL: source DB schema dump (from writeExtension) ---
- sourceSQL := `CREATE SCHEMA IF NOT EXISTS "tecbatch";
-
-CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "tecbatch" VERSION '1.6';
-
-COMMENT ON EXTENSION "pg_trgm" IS 'text similarity measurement and index searching based on trigrams';
-
-CREATE OR REPLACE FUNCTION "tecbatch"."contains_one_of"(str character varying, match_strs character varying[], match_case boolean)
- RETURNS boolean
- LANGUAGE plpgsql
- IMMUTABLE
-AS $function$
-DECLARE
- result boolean := false;
-BEGIN
- RETURN result;
-END;
-$function$;`
-
- // Parse source SQL (this is the buggy path)
- newMetadata, err := GetDatabaseMetadata(sourceSQL)
- require.NoError(t, err)
-
- // Build model objects for differ
- oldModel := model.NewDatabaseMetadata(oldMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- newModel := model.NewDatabaseMetadata(newMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- // Run differ
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, oldModel, newModel)
- require.NoError(t, err)
-
- // Generate migration DDL
- migration, err := generateMigration(diff)
- require.NoError(t, err)
-
- // The source and target have identical extensions → migration should be empty
- // (or at most contain the function CREATE which is expected).
- require.NotContains(t, migration, "DROP EXTENSION",
- "BUG: Schema Sync incorrectly drops pg_trgm extension that exists in both databases")
- require.NotContains(t, migration, `CREATE SCHEMA IF NOT EXISTS "pg_trgm"`,
- "BUG: Schema Sync creates phantom pg_trgm schema from COMMENT ON EXTENSION")
- require.NotContains(t, migration, "CREATE EXTENSION",
- "BUG: Schema Sync recreates pg_trgm extension even though it's identical")
-
- // Log the migration for debugging if any assertions above fail
- if t.Failed() {
- t.Logf("Generated migration (should be empty or function-only):\n%s", migration)
- }
- })
-}
diff --git a/backend/plugin/schema/pg/get_database_definition.go b/backend/plugin/schema/pg/get_database_definition.go
index dee00c5cd0..f2873f7898 100644
--- a/backend/plugin/schema/pg/get_database_definition.go
+++ b/backend/plugin/schema/pg/get_database_definition.go
@@ -6,15 +6,14 @@ import (
"slices"
"strings"
- "github.com/antlr4-go/antlr/v4"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/postgresql"
+ omnipg "github.com/bytebase/omni/pg"
+ "github.com/bytebase/omni/pg/ast"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
"github.com/bytebase/bytebase/backend/plugin/schema"
)
@@ -37,14 +36,16 @@ SET row_security = off;
)
func init() {
- schema.RegisterGetDatabaseDefinition(storepb.Engine_POSTGRES, GetDatabaseDefinition)
- schema.RegisterGetSchemaDefinition(storepb.Engine_POSTGRES, GetSchemaDefinition)
- schema.RegisterGetTableDefinition(storepb.Engine_POSTGRES, GetTableDefinition)
- schema.RegisterGetViewDefinition(storepb.Engine_POSTGRES, GetViewDefinition)
- schema.RegisterGetMaterializedViewDefinition(storepb.Engine_POSTGRES, GetMaterializedViewDefinition)
- schema.RegisterGetFunctionDefinition(storepb.Engine_POSTGRES, GetFunctionDefinition)
- schema.RegisterGetSequenceDefinition(storepb.Engine_POSTGRES, GetSequenceDefinition)
- schema.RegisterGetMultiFileDatabaseDefinition(storepb.Engine_POSTGRES, GetMultiFileDatabaseDefinition)
+ for _, engine := range []storepb.Engine{storepb.Engine_POSTGRES, storepb.Engine_COCKROACHDB} {
+ schema.RegisterGetDatabaseDefinition(engine, GetDatabaseDefinition)
+ schema.RegisterGetSchemaDefinition(engine, GetSchemaDefinition)
+ schema.RegisterGetTableDefinition(engine, GetTableDefinition)
+ schema.RegisterGetViewDefinition(engine, GetViewDefinition)
+ schema.RegisterGetMaterializedViewDefinition(engine, GetMaterializedViewDefinition)
+ schema.RegisterGetFunctionDefinition(engine, GetFunctionDefinition)
+ schema.RegisterGetSequenceDefinition(engine, GetSequenceDefinition)
+ schema.RegisterGetMultiFileDatabaseDefinition(engine, GetMultiFileDatabaseDefinition)
+ }
}
func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (string, error) {
@@ -2240,60 +2241,26 @@ func writeFunction(out io.Writer, schema string, function *storepb.FunctionMetad
return nil
}
-// functionExtractor walks a PostgreSQL parse tree and captures the first
-// CREATE FUNCTION/PROCEDURE statement it encounters.
-type functionExtractor struct {
- parser.BasePostgreSQLParserListener
- result **parser.CreatefunctionstmtContext
-}
-
-func (e *functionExtractor) EnterCreatefunctionstmt(ctx *parser.CreatefunctionstmtContext) {
- if e.result != nil && *e.result == nil {
- *e.result = ctx
- }
-}
-
-// isDefinitionProcedure checks if the definition string represents a PROCEDURE (not a FUNCTION)
-// Returns true if it's a PROCEDURE, false if it's a FUNCTION
-// This function uses AST-based parsing for robust detection
+// isDefinitionProcedure checks if the definition string represents a PROCEDURE (not a FUNCTION).
+// Uses omni parser: procedures have a DefElem{Defname: "isProcedure"} in Options.
func isDefinitionProcedure(definition string) bool {
if definition == "" {
return false
}
-
- // Parse the definition to get AST
- parseResults, err := pgparser.ParsePostgreSQL(definition)
- if err != nil {
- // If parsing fails, fall back to string-based detection
- // This should rarely happen for valid definitions
- upperDef := strings.ToUpper(definition)
- return strings.Contains(upperDef, " PROCEDURE ") ||
- strings.HasPrefix(upperDef, "CREATE PROCEDURE") ||
- strings.HasPrefix(upperDef, "CREATE OR REPLACE PROCEDURE")
- }
-
- // For function/procedure definition, we expect exactly one statement
- if len(parseResults) != 1 {
+ stmts, err := omnipg.Parse(definition)
+ if err != nil || len(stmts) == 0 {
return false
}
-
- tree := parseResults[0].Tree
- if tree == nil {
+ fn, ok := stmts[0].AST.(*ast.CreateFunctionStmt)
+ if !ok || fn == nil || fn.Options == nil {
return false
}
-
- // Walk the AST to find CREATE FUNCTION/PROCEDURE statement
- var result *parser.CreatefunctionstmtContext
- extractor := &functionExtractor{result: &result}
- antlr.NewParseTreeWalker().Walk(extractor, tree)
-
- if result == nil {
- return false
+ for _, item := range fn.Options.Items {
+ if def, ok := item.(*ast.DefElem); ok && def.Defname == "isProcedure" {
+ return true
+ }
}
-
- // Check if it's a PROCEDURE by examining the AST node
- // CreatefunctionstmtContext has both FUNCTION() and PROCEDURE() methods
- return result.PROCEDURE() != nil
+ return false
}
func writeFunctionComment(out io.Writer, schema string, function *storepb.FunctionMetadata) error {
@@ -3246,15 +3213,14 @@ func writeIndexInternal(out io.Writer, schema string, table string, index *store
}
// Write INCLUDE and WHERE clauses from the full index definition (from pg_get_indexdef).
- // SQL syntax order: (keys) INCLUDE (cols) WHERE (predicate)
if index.Definition != "" {
- comparer := &PostgreSQLIndexComparer{}
- if includeClause := comparer.ExtractIncludeClauseFromIndexDef(index.Definition); includeClause != "" {
+ includeClause, whereClause := extractIndexClauses(index.Definition)
+ if includeClause != "" {
if _, err := fmt.Fprintf(out, " %s", includeClause); err != nil {
return err
}
}
- if whereClause := comparer.ExtractWhereClauseFromIndexDef(index.Definition); whereClause != "" {
+ if whereClause != "" {
if _, err := fmt.Fprintf(out, " WHERE %s", whereClause); err != nil {
return err
}
@@ -4268,3 +4234,66 @@ func writeTriggerCommentSDL(out io.Writer, schemaName, tableName string, trigger
_, err := io.WriteString(out, "\n\n")
return err
}
+
+// extractIndexClauses parses a CREATE INDEX statement using the omni parser and
+// extracts the INCLUDE clause and WHERE clause as strings.
+func extractIndexClauses(definition string) (includeClause, whereClause string) {
+ stmts, err := omnipg.Parse(definition)
+ if err != nil || len(stmts) == 0 {
+ return "", ""
+ }
+ idx, ok := stmts[0].AST.(*ast.IndexStmt)
+ if !ok || idx == nil {
+ return "", ""
+ }
+
+ // Extract INCLUDE columns using Loc to preserve original quoting.
+ if idx.IndexIncludingParams != nil {
+ var cols []string
+ for _, item := range idx.IndexIncludingParams.Items {
+ if elem, ok := item.(*ast.IndexElem); ok && elem.Loc.Start >= 0 && elem.Loc.End > elem.Loc.Start {
+ cols = append(cols, definition[elem.Loc.Start:elem.Loc.End])
+ }
+ }
+ if len(cols) > 0 {
+ includeClause = "INCLUDE (" + strings.Join(cols, ", ") + ")"
+ }
+ }
+
+ // Extract WHERE clause using AST node Loc for precise extraction.
+ if idx.WhereClause != nil {
+ loc := ast.NodeLoc(idx.WhereClause)
+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(definition) {
+ raw := definition[loc.Start:loc.End]
+ // Strip outer parentheses if the entire predicate is wrapped.
+ if strings.HasPrefix(raw, "(") && strings.HasSuffix(raw, ")") {
+ inner := raw[1 : len(raw)-1]
+ // Only strip if parentheses are balanced.
+ if isBalancedParens(inner) {
+ raw = inner
+ }
+ }
+ whereClause = strings.TrimSpace(raw)
+ }
+ }
+
+ return includeClause, whereClause
+}
+
+// isBalancedParens checks if a string has balanced parentheses.
+func isBalancedParens(s string) bool {
+ depth := 0
+ for _, c := range s {
+ switch c {
+ case '(':
+ depth++
+ case ')':
+ depth--
+ if depth < 0 {
+ return false
+ }
+ default:
+ }
+ }
+ return depth == 0
+}
diff --git a/backend/plugin/schema/pg/get_database_metadata.go b/backend/plugin/schema/pg/get_database_metadata.go
deleted file mode 100644
index 2faefc77c5..0000000000
--- a/backend/plugin/schema/pg/get_database_metadata.go
+++ /dev/null
@@ -1,2923 +0,0 @@
-package pg
-
-import (
- "context"
- "fmt"
- "slices"
- "strings"
-
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/postgresql"
- "github.com/pkg/errors"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
- "github.com/bytebase/bytebase/backend/plugin/schema"
- "github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"
- "github.com/bytebase/bytebase/backend/store/model"
-)
-
-func init() {
- schema.RegisterGetDatabaseMetadata(storepb.Engine_POSTGRES, GetDatabaseMetadata)
- schema.RegisterGetDatabaseMetadata(storepb.Engine_COCKROACHDB, GetDatabaseMetadata)
-}
-
-// GetDatabaseMetadata parses the SQL schema text and returns the database metadata.
-func GetDatabaseMetadata(schemaText string) (*storepb.DatabaseSchemaMetadata, error) {
- parseResults, err := pgparser.ParsePostgreSQL(schemaText)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to parse PostgreSQL schema")
- }
-
- extractor := &metadataExtractor{
- currentDatabase: "",
- currentSchema: "public",
- schemas: make(map[string]*storepb.SchemaMetadata),
- tables: make(map[tableKey]*storepb.TableMetadata),
- partitionTables: make(map[tableKey]bool),
- partitionExpressions: make(map[tableKey]string),
- partitionTypes: make(map[tableKey]storepb.TablePartitionMetadata_Type),
- sequences: make(map[string]*storepb.SequenceMetadata),
- extensions: make(map[string]*storepb.ExtensionMetadata),
- expressionComparer: ast.NewPostgreSQLExpressionComparer(),
- triggers: make(map[string][]*storepb.TriggerMetadata),
- }
-
- // Always ensure public schema exists
- extractor.getOrCreateSchema("public")
-
- // Walk all parsed statements
- for _, parseResult := range parseResults {
- if parseResult.Tree != nil {
- antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)
- }
- }
-
- if extractor.err != nil {
- return nil, extractor.err
- }
-
- // Build the final metadata structure
- schemaMetadata := &storepb.DatabaseSchemaMetadata{
- Name: extractor.currentDatabase,
- }
-
- // Sort schemas for consistent output
- var schemaNames []string
- for name := range extractor.schemas {
- schemaNames = append(schemaNames, name)
- }
- slices.Sort(schemaNames)
-
- for _, schemaName := range schemaNames {
- schema := extractor.schemas[schemaName]
-
- // Filter out any tables that are actually materialized views
- // This handles the case where materialized views are incorrectly classified as tables
- filteredTables := []*storepb.TableMetadata{}
- for _, table := range schema.Tables {
- // Check if this table name exists as a materialized view
- isMaterializedView := false
- for _, mv := range schema.MaterializedViews {
- if mv.Name == table.Name {
- isMaterializedView = true
- break
- }
- }
-
- // Only include in tables if it's not a materialized view
- if !isMaterializedView {
- filteredTables = append(filteredTables, table)
- }
- }
- schema.Tables = filteredTables
-
- // Assign triggers to their respective tables
- for _, table := range filteredTables {
- tableKey := fmt.Sprintf("%s.%s", schemaName, table.Name)
- if triggers, exists := extractor.triggers[tableKey]; exists {
- table.Triggers = triggers
- }
- }
-
- // Also assign triggers to materialized views if any
- for _, mv := range schema.MaterializedViews {
- tableKey := fmt.Sprintf("%s.%s", schemaName, mv.Name)
- if triggers, exists := extractor.triggers[tableKey]; exists {
- mv.Triggers = triggers
- }
- }
-
- // Add sequences to the schema (including SERIAL-generated sequences)
- var sequencesForSchema []*storepb.SequenceMetadata
- for sequenceKey, sequence := range extractor.sequences {
- // Parse sequence key format: "schemaname.sequencename"
- parts := strings.SplitN(sequenceKey, ".", 2)
- if len(parts) == 2 && parts[0] == schemaName {
- sequencesForSchema = append(sequencesForSchema, sequence)
- }
- }
- // Sort sequences for consistent output
- slices.SortFunc(sequencesForSchema, func(a, b *storepb.SequenceMetadata) int {
- return strings.Compare(a.Name, b.Name)
- })
- schema.Sequences = sequencesForSchema
-
- schemaMetadata.Schemas = append(schemaMetadata.Schemas, schema)
- }
-
- // Add extensions
- var extensionNames []string
- for name := range extractor.extensions {
- extensionNames = append(extensionNames, name)
- }
- slices.Sort(extensionNames)
-
- for _, name := range extensionNames {
- schemaMetadata.Extensions = append(schemaMetadata.Extensions, extractor.extensions[name])
- }
-
- // Extract view dependencies after all metadata is collected
- extractViewDependencies(schemaMetadata)
-
- return schemaMetadata, nil
-}
-
-type tableKey struct {
- schema string
- table string
-}
-
-// metadataExtractor walks the parse tree and extracts metadata
-type metadataExtractor struct {
- *parser.BasePostgreSQLParserListener
-
- currentDatabase string
- currentSchema string
- schemas map[string]*storepb.SchemaMetadata
- tables map[tableKey]*storepb.TableMetadata
- partitionTables map[tableKey]bool
- partitionExpressions map[tableKey]string
- partitionTypes map[tableKey]storepb.TablePartitionMetadata_Type
- sequences map[string]*storepb.SequenceMetadata
- extensions map[string]*storepb.ExtensionMetadata
- expressionComparer ast.ExpressionComparer
- triggers map[string][]*storepb.TriggerMetadata // Map from table key to triggers
- err error
-}
-
-// Helper function to get or create schema
-func (e *metadataExtractor) getOrCreateSchema(schemaName string) *storepb.SchemaMetadata {
- if schemaName == "" {
- schemaName = "public"
- }
-
- if schema, exists := e.schemas[schemaName]; exists {
- return schema
- }
-
- schema := &storepb.SchemaMetadata{
- Name: schemaName,
- Tables: []*storepb.TableMetadata{},
- Views: nil,
- MaterializedViews: nil,
- Procedures: nil,
- Functions: nil,
- Sequences: nil,
- EnumTypes: nil,
- }
- e.schemas[schemaName] = schema
- return schema
-}
-
-// Helper function to get or create table
-func (e *metadataExtractor) getOrCreateTable(schemaName, tableName string) *storepb.TableMetadata {
- key := tableKey{
- schema: schemaName,
- table: tableName,
- }
-
- if table, exists := e.tables[key]; exists {
- return table
- }
-
- table := &storepb.TableMetadata{
- Name: tableName,
- Columns: []*storepb.ColumnMetadata{},
- Indexes: []*storepb.IndexMetadata{},
- ForeignKeys: nil,
- CheckConstraints: nil,
- Triggers: nil,
- Partitions: nil,
- }
-
- // Only add to schema's table list if it's not a partition table
- if !e.partitionTables[key] {
- schema := e.getOrCreateSchema(schemaName)
- schema.Tables = append(schema.Tables, table)
- }
- e.tables[key] = table
-
- return table
-}
-
-// Helper function to find materialized view in a schema
-func (e *metadataExtractor) findMaterializedView(schemaName, viewName string) *storepb.MaterializedViewMetadata {
- schema := e.getOrCreateSchema(schemaName)
- if schema.MaterializedViews == nil {
- return nil
- }
-
- for _, mv := range schema.MaterializedViews {
- if mv.Name == viewName {
- return mv
- }
- }
- return nil
-}
-
-// EnterCreateschemastmt is called when entering a create schema statement
-func (e *metadataExtractor) EnterCreateschemastmt(ctx *parser.CreateschemastmtContext) {
- if e.err != nil {
- return
- }
-
- // Try to get schema name directly from Colid first
- if ctx.Colid() != nil {
- schemaName := pgparser.NormalizePostgreSQLColid(ctx.Colid())
- e.getOrCreateSchema(schemaName)
- } else if ctx.Optschemaname() != nil && ctx.Optschemaname().Colid() != nil {
- schemaName := pgparser.NormalizePostgreSQLColid(ctx.Optschemaname().Colid())
- e.getOrCreateSchema(schemaName)
- }
-}
-
-// EnterCreatestmt is called when entering a create table statement
-func (e *metadataExtractor) EnterCreatestmt(ctx *parser.CreatestmtContext) {
- if e.err != nil {
- return
- }
-
- if ctx.Qualified_name(0) == nil {
- return
- }
-
- qualifiedName := ctx.Qualified_name(0)
- schemaName, tableName := e.extractSchemaAndTableName(qualifiedName)
-
- // Check if this is a partition table (CREATE TABLE ... PARTITION OF ...)
- // We'll check if there's a second qualified_name which would be the parent table
- if ctx.Qualified_name(1) != nil && ctx.PARTITION() != nil && ctx.OF() != nil {
- // Mark this table as a partition
- key := tableKey{schema: schemaName, table: tableName}
- e.partitionTables[key] = true
-
- // Get the parent table and add this partition to it
- parentSchema, parentTable := e.extractSchemaAndTableName(ctx.Qualified_name(1))
- parentTableMetadata := e.getOrCreateTable(parentSchema, parentTable)
-
- // Create partition metadata
- partition := &storepb.TablePartitionMetadata{
- Name: tableName,
- }
-
- // Extract FOR VALUES clause if present
- if ctx.Partitionboundspec() != nil {
- partition.Value = e.extractPartitionBoundSpec(ctx.Partitionboundspec())
- }
-
- // Get the parent table's partition expression and type
- parentKey := tableKey{schema: parentSchema, table: parentTable}
- if expr, ok := e.partitionExpressions[parentKey]; ok {
- partition.Expression = expr
- }
- if partitionType, ok := e.partitionTypes[parentKey]; ok {
- partition.Type = partitionType
- }
-
- if parentTableMetadata.Partitions == nil {
- parentTableMetadata.Partitions = []*storepb.TablePartitionMetadata{}
- }
- parentTableMetadata.Partitions = append(parentTableMetadata.Partitions, partition)
-
- // Don't continue processing this table as a regular table
- return
- }
-
- // Create the table (it's not a partition table)
- tableMetadata := e.getOrCreateTable(schemaName, tableName)
-
- // Extract table elements (columns, constraints)
- if tableElementList := ctx.Opttableelementlist(); tableElementList != nil {
- e.extractTableElements(tableElementList, tableMetadata, schemaName)
- }
-
- // Extract partition info and store the expression and type
- if partitionSpec := ctx.Optpartitionspec(); partitionSpec != nil {
- // Store the partition expression and type for this table
- key := tableKey{schema: schemaName, table: tableName}
- e.partitionExpressions[key] = e.extractPartitionExpression(partitionSpec)
- e.partitionTypes[key] = e.extractPartitionType(partitionSpec)
- }
-}
-
-// extractSchemaAndTableName extracts schema and table name from qualified name
-func (e *metadataExtractor) extractSchemaAndTableName(ctx parser.IQualified_nameContext) (string, string) {
- if ctx == nil {
- return e.currentSchema, ""
- }
-
- // Use NormalizePostgreSQLQualifiedName which properly handles quoted identifiers
- // This function removes quotes and handles case sensitivity correctly
- parts := pgparser.NormalizePostgreSQLQualifiedName(ctx)
- if len(parts) == 1 {
- return e.currentSchema, parts[0]
- } else if len(parts) == 2 {
- return parts[0], parts[1]
- }
- return e.currentSchema, ""
-}
-
-// extractTableElements extracts columns and constraints from table elements
-func (e *metadataExtractor) extractTableElements(ctx parser.IOpttableelementlistContext, table *storepb.TableMetadata, schemaName string) {
- if ctx == nil {
- return
- }
-
- // Get the table element list
- tableElementList := ctx.Tableelementlist()
- if tableElementList == nil {
- return
- }
-
- // Process all table elements
- for _, tableElement := range tableElementList.AllTableelement() {
- if tableElement == nil {
- continue
- }
-
- // Handle column definitions
- if columnDef := tableElement.ColumnDef(); columnDef != nil {
- e.extractColumnDef(columnDef, table, schemaName)
- }
-
- // Handle table constraints
- if tableConstraint := tableElement.Tableconstraint(); tableConstraint != nil {
- e.extractTableConstraint(tableConstraint, table, schemaName)
- }
- }
-}
-
-// extractColumnDef extracts column definition
-func (e *metadataExtractor) extractColumnDef(ctx parser.IColumnDefContext, table *storepb.TableMetadata, schemaName string) {
- if ctx == nil {
- return
- }
-
- column := &storepb.ColumnMetadata{
- Nullable: true, // Default to nullable
- }
-
- // Extract column name
- if ctx.Colid() != nil {
- column.Name = pgparser.NormalizePostgreSQLColid(ctx.Colid())
- }
-
- // Extract data type
- if ctx.Typename() != nil {
- // Get the raw type name to check for SERIAL
- rawTypeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Typename())
- rawTypeName = strings.ToLower(rawTypeName)
-
- // Handle SERIAL types completely - PostgreSQL expands SERIAL into integer + sequence + default
- switch rawTypeName {
- case "serial", "serial4":
- column.Nullable = false
- column.Type = "integer"
- // Create sequence and set default value
- e.createSerialSequenceAndDefault(column, table, schemaName)
- case "bigserial", "serial8":
- column.Nullable = false
- column.Type = "bigint"
- // Create sequence and set default value
- e.createSerialSequenceAndDefault(column, table, schemaName)
- case "smallserial", "serial2":
- column.Nullable = false
- column.Type = "smallint"
- // Create sequence and set default value
- e.createSerialSequenceAndDefault(column, table, schemaName)
- default:
- column.Type = e.extractTypeNameWithSchema(ctx.Typename(), schemaName)
- }
- }
-
- // Extract column constraints
- if colquallist := ctx.Colquallist(); colquallist != nil {
- for _, colConstraint := range colquallist.AllColconstraint() {
- if colConstraint != nil && colConstraint.Colconstraintelem() != nil {
- // Extract constraint name if present
- var constraintName string
- if colConstraint.CONSTRAINT() != nil && colConstraint.Name() != nil {
- constraintName = pgparser.NormalizePostgreSQLName(colConstraint.Name())
- }
- e.extractColumnConstraint(colConstraint.Colconstraintelem(), column, table, constraintName, schemaName)
- }
- }
- }
-
- table.Columns = append(table.Columns, column)
-}
-
-// createSerialSequenceAndDefault creates a sequence for SERIAL columns and sets the default value
-func (e *metadataExtractor) createSerialSequenceAndDefault(column *storepb.ColumnMetadata, table *storepb.TableMetadata, schemaName string) {
- // Generate sequence name following PostgreSQL convention: tablename_columnname_seq
- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)
-
- // Determine sequence data type based on column type
- // SERIAL (integer) → bigint sequence (PostgreSQL standard)
- // BIGSERIAL (bigint) → bigint sequence
- // SMALLSERIAL (smallint) → bigint sequence
- // All sequences in PostgreSQL are bigint by default
- dataType := "bigint"
-
- // Create the sequence metadata with proper defaults matching PostgreSQL behavior
- sequence := &storepb.SequenceMetadata{
- Name: sequenceName,
- DataType: dataType,
- Start: "1",
- Increment: "1",
- MinValue: "1",
- MaxValue: "9223372036854775807", // bigint max value
- Cycle: false,
- CacheSize: "1",
- OwnerTable: table.Name,
- OwnerColumn: column.Name,
- }
-
- // Add sequence to the global sequences map using fully qualified name
- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)
- e.sequences[sequenceKey] = sequence
-
- // Set the column default value to use nextval()
- // PostgreSQL stores this as: nextval('schema.sequence_name'::regclass)
- column.Default = fmt.Sprintf("nextval('%s.%s'::regclass)", schemaName, sequenceName)
-}
-
-// extractTypeName extracts the type name from typename context
-func extractTypeName(ctx parser.ITypenameContext) string {
- if ctx == nil {
- return ""
- }
- // Get the full text representation of the type
- typeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- // Convert to lowercase to match sync.go output
- typeName = strings.ToLower(typeName)
-
- return NormalizePostgreSQLType(typeName)
-}
-
-// getPostgreSQLArrayTypeName maps base types to their PostgreSQL internal array type names
-// Based on PostgreSQL system catalog pg_type where array types start with underscore
-func getPostgreSQLArrayTypeName(baseType string) string {
- // Normalize the base type first to handle aliases
- normalizedBase := strings.ToLower(strings.TrimSpace(baseType))
-
- // Map of base types to their PostgreSQL internal array type names
- arrayTypeMap := map[string]string{
- // Integer types
- "int": "_int4",
- "int4": "_int4",
- "integer": "_int4",
- "int2": "_int2",
- "smallint": "_int2",
- "int8": "_int8",
- "bigint": "_int8",
-
- // Floating point types
- "real": "_float4",
- "float4": "_float4",
- "double precision": "_float8",
- "float8": "_float8",
-
- // Character types
- "text": "_text",
- "varchar": "_varchar",
- "character varying": "_varchar",
- "char": "_bpchar", // PostgreSQL uses bpchar internally for char
- "character": "_bpchar", // PostgreSQL uses bpchar internally for character
- "bpchar": "_bpchar",
- "name": "_name",
-
- // Boolean type
- "bool": "_bool",
- "boolean": "_bool",
-
- // Numeric types
- "numeric": "_numeric",
- "decimal": "_numeric",
-
- // Date/time types
- "date": "_date",
- "time": "_time",
- "time without time zone": "_time",
- "time with time zone": "_timetz",
- "timetz": "_timetz",
- "timestamp": "_timestamp",
- "timestamp without time zone": "_timestamp",
- "timestamp with time zone": "_timestamptz",
- "timestamptz": "_timestamptz",
- "interval": "_interval",
-
- // Binary and other types
- "bytea": "_bytea",
- "uuid": "_uuid",
- "json": "_json",
- "jsonb": "_jsonb",
- "xml": "_xml",
- "money": "_money",
- "inet": "_inet",
- "cidr": "_cidr",
- "macaddr": "_macaddr",
- "macaddr8": "_macaddr8",
-
- // Geometric types
- "point": "_point",
- "line": "_line",
- "lseg": "_lseg",
- "box": "_box",
- "path": "_path",
- "polygon": "_polygon",
- "circle": "_circle",
-
- // Full-text search types
- "tsvector": "_tsvector",
- "tsquery": "_tsquery",
-
- // Range types
- "int4range": "_int4range",
- "int8range": "_int8range",
- "numrange": "_numrange",
- "tsrange": "_tsrange",
- "tstzrange": "_tstzrange",
- "daterange": "_daterange",
-
- // Multi-range types (PostgreSQL 14+)
- "int4multirange": "_int4multirange",
- "int8multirange": "_int8multirange",
- "nummultirange": "_nummultirange",
- "tsmultirange": "_tsmultirange",
- "tstzmultirange": "_tstzmultirange",
- "datemultirange": "_datemultirange",
-
- // Bit string types
- "bit": "_bit",
- "bit varying": "_varbit",
- "varbit": "_varbit",
-
- // Object identifier types
- "oid": "_oid",
- "regproc": "_regproc",
- "regprocedure": "_regprocedure",
- "regoper": "_regoper",
- "regoperator": "_regoperator",
- "regclass": "_regclass",
- "regtype": "_regtype",
- "regconfig": "_regconfig",
- "regdictionary": "_regdictionary",
- "regnamespace": "_regnamespace",
- "regrole": "_regrole",
- "regcollation": "_regcollation",
-
- // Other system types
- "tid": "_tid",
- "xid": "_xid",
- "xid8": "_xid8",
- "cid": "_cid",
- "pg_lsn": "_pg_lsn",
- "record": "_record",
- "cstring": "_cstring",
- "refcursor": "_refcursor",
- "jsonpath": "_jsonpath",
- "txid_snapshot": "_txid_snapshot",
- "pg_snapshot": "_pg_snapshot",
- "gtsvector": "_gtsvector",
- "aclitem": "_aclitem",
- "int2vector": "_int2vector",
- "oidvector": "_oidvector",
- }
-
- // Handle types with precision/scale (e.g., varchar(255), numeric(10,2), character varying(255))
- if idx := strings.Index(normalizedBase, "("); idx != -1 {
- baseTypeWithoutPrecision := normalizedBase[:idx]
- if arrayType, exists := arrayTypeMap[baseTypeWithoutPrecision]; exists {
- return arrayType
- }
- }
-
- // Direct lookup
- if arrayType, exists := arrayTypeMap[normalizedBase]; exists {
- return arrayType
- }
-
- // For serial types, they should not have array equivalents in DDL,
- // but if they do, map to their underlying integer array types
- switch normalizedBase {
- case "serial", "serial4":
- return "_int4"
- case "bigserial", "serial8":
- return "_int8"
- case "smallserial", "serial2":
- return "_int2"
- default:
- // For non-serial types, proceed with fallback logic
- }
-
- // Return empty string if no mapping found - caller will use fallback logic
- return ""
-}
-
-// normalizePostgreSQLType normalizes PostgreSQL type names to match sync.go output
-func NormalizePostgreSQLType(typeName string) string {
- // Remove extra whitespace
- typeName = strings.TrimSpace(typeName)
-
- // Handle array types FIRST before any other conversions
- // text[] -> _text, int[] -> _int4, varchar(255)[] -> _varchar, etc.
- // PostgreSQL treats multi-dimensional arrays the same as single dimension
- if strings.HasSuffix(typeName, "[]") {
- // Remove all array dimensions (int[][], int[] both become int)
- baseType := typeName
- for strings.HasSuffix(baseType, "[]") {
- baseType = baseType[:len(baseType)-2]
- }
-
- // Map base type to PostgreSQL's internal array type name
- arrayType := getPostgreSQLArrayTypeName(baseType)
- if arrayType != "" {
- return arrayType
- }
-
- // For unknown types, use the old logic as fallback
- normalizedBase := NormalizePostgreSQLType(baseType)
- return "_" + normalizedBase
- }
-
- // Convert common type variations to match sync.go output
- // varchar(n) -> character varying(n)
- if strings.HasPrefix(typeName, "varchar(") {
- return "character varying" + typeName[7:]
- }
- if typeName == "varchar" {
- return "character varying"
- }
-
- // SERIAL types are stored as integer/bigint in the catalog
- if typeName == "serial" || typeName == "serial4" {
- return "integer"
- }
- if typeName == "bigserial" || typeName == "serial8" {
- return "bigint"
- }
- if typeName == "smallserial" || typeName == "serial2" {
- return "smallint"
- }
-
- // Handle timestamp without explicit timezone
- if typeName == "timestamp" {
- return "timestamp(6) without time zone" // PostgreSQL default precision is 6
- }
- // Handle timestamp with time zone (no precision specified)
- if typeName == "timestamp with time zone" {
- return "timestamp(6) with time zone" // PostgreSQL default precision is 6
- }
- // Handle timestamp with precision (preserve precision but add timezone info)
- if strings.HasPrefix(typeName, "timestamp(") && strings.HasSuffix(typeName, ")") {
- // Extract precision part: timestamp(3) -> (3)
- precision := typeName[9:] // Get "(3)" part
- return "timestamp" + precision + " without time zone"
- }
- // Handle time without explicit timezone (PostgreSQL default)
- if typeName == "time" {
- return "time(6) without time zone" // PostgreSQL default precision is 6
- }
- // Handle time with time zone (no precision specified)
- if typeName == "time with time zone" {
- return "time(6) with time zone" // PostgreSQL default precision is 6
- }
- // Handle time with precision (preserve precision but add timezone info)
- if strings.HasPrefix(typeName, "time(") && strings.HasSuffix(typeName, ")") {
- // Extract precision part: time(6) -> (6)
- precision := typeName[4:] // Get "(6)" part
- return "time" + precision + " without time zone"
- }
-
- // Handle decimal -> numeric (preserve precision information)
- if strings.HasPrefix(typeName, "decimal(") {
- // Convert decimal(p,s) to numeric(p,s) while preserving precision
- // Also normalize spacing to match sync.go format (remove spaces around commas)
- precisionPart := typeName[7:] // Get "(p, s)" or "(p,s)" part
- precisionPart = strings.ReplaceAll(precisionPart, " ", "") // Remove all spaces to match sync.go format
- return "numeric" + precisionPart
- }
- if typeName == "decimal" {
- return "numeric"
- }
-
- // Normalize common type aliases to match PostgreSQL catalog output
- switch typeName {
- case "int", "int4":
- return "integer"
- case "int2":
- return "smallint"
- case "int8":
- return "bigint"
- case "float4":
- return "real"
- case "float8":
- return "double precision"
- case "bool":
- return "boolean"
- case "char":
- return "character"
- case "timestamptz":
- return "timestamp(6) with time zone" // PostgreSQL default precision is 6
- case "timetz":
- return "time(6) with time zone" // PostgreSQL default precision is 6
- case "varbit":
- return "bit varying"
- default:
- // Return type as-is for unrecognized types
- }
-
- // Handle specific length specifications for character types
- if strings.HasPrefix(typeName, "char(") {
- return "character" + typeName[4:]
- }
-
- // Handle varbit with precision: varbit(n) -> bit varying(n)
- if strings.HasPrefix(typeName, "varbit(") {
- return "bit varying" + typeName[6:] // Replace "varbit" with "bit varying"
- }
-
- // For all remaining types with precision/scale parameters, normalize spacing to match sync.go format
- // This handles cases like "numeric(10, 2)" -> "numeric(10,2)" to match database sync output
- if strings.Contains(typeName, "(") && strings.Contains(typeName, ")") {
- // Remove spaces around commas in type specifications to match sync.go format
- typeName = strings.ReplaceAll(typeName, ", ", ",")
- typeName = strings.ReplaceAll(typeName, " ,", ",")
- }
-
- return typeName
-}
-
-// extractTypeNameWithSchema extracts the type name and adds schema prefix for custom types
-func (*metadataExtractor) extractTypeNameWithSchema(ctx parser.ITypenameContext, schemaName string) string {
- if ctx == nil {
- return ""
- }
-
- // Get the raw type name first (without normalization)
- rawTypeName := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- rawTypeName = strings.ToLower(strings.TrimSpace(rawTypeName))
-
- // Check if this is a built-in PostgreSQL type or an array type before normalization
- // Array types should always go through normalization regardless of their base type
- if isBuiltInType(rawTypeName) || isBuiltInType(NormalizePostgreSQLType(rawTypeName)) || strings.HasSuffix(rawTypeName, "[]") {
- // For built-in types and array types, return the normalized version
- return NormalizePostgreSQLType(rawTypeName)
- }
-
- // Check if the type already has a schema prefix
- if strings.Contains(rawTypeName, ".") {
- // It's already schema-qualified, just normalize it
- return NormalizePostgreSQLType(rawTypeName)
- }
-
- // For custom types (like enums), add the schema prefix to the raw type name
- return fmt.Sprintf("%s.%s", schemaName, rawTypeName)
-}
-
-// isBuiltInType checks if a type is a built-in PostgreSQL type
-func isBuiltInType(typeName string) bool {
- // Handle array types first (before removing precision info)
- if strings.HasSuffix(typeName, "[]") {
- // Remove all array dimensions (int[][], int[] both should be recognized)
- arrayBase := typeName
- for strings.HasSuffix(arrayBase, "[]") {
- arrayBase = arrayBase[:len(arrayBase)-2]
- }
- // Recursively check if the base type is built-in
- return isBuiltInType(arrayBase)
- }
- if strings.HasPrefix(typeName, "_") {
- // PostgreSQL internal array notation _typename
- arrayBase := typeName[1:]
- return isBuiltInType(arrayBase)
- }
-
- // Remove any precision/scale information for checking
- baseType := typeName
- if idx := strings.Index(typeName, "("); idx != -1 {
- baseType = typeName[:idx]
- }
-
- // Common built-in PostgreSQL types including both aliases and canonical forms
- builtInTypes := map[string]bool{
- // Integer types
- "integer": true, "int": true, "int4": true,
- "bigint": true, "int8": true,
- "smallint": true, "int2": true,
- "serial": true, "serial4": true,
- "bigserial": true, "serial8": true, "smallserial": true, "serial2": true,
-
- // Numeric types
- "numeric": true, "decimal": true,
- "real": true, "float4": true,
- "double precision": true, "float8": true,
- "money": true,
-
- // Character types
- "character": true, "char": true,
- "character varying": true, "varchar": true,
- "text": true,
- "name": true,
-
- // Boolean
- "boolean": true, "bool": true,
-
- // Date/time types
- "date": true,
- "time": true, "time without time zone": true, "time with time zone": true, "timetz": true,
- "timestamp": true, "timestamp without time zone": true, "timestamp with time zone": true, "timestamptz": true,
- "interval": true,
-
- // UUID
- "uuid": true,
-
- // JSON types
- "json": true, "jsonb": true,
-
- // XML
- "xml": true,
-
- // Binary data
- "bytea": true,
-
- // Bit string types
- "bit": true, "bit varying": true, "varbit": true,
-
- // Network address types
- "inet": true, "cidr": true, "macaddr": true, "macaddr8": true,
-
- // Geometric types
- "point": true, "line": true, "lseg": true, "box": true,
- "path": true, "polygon": true, "circle": true,
-
- // Full-text search types
- "tsquery": true, "tsvector": true,
-
- // Range types
- "int4range": true, "int8range": true, "numrange": true,
- "tsrange": true, "tstzrange": true, "daterange": true,
-
- // Other types
- "pg_lsn": true,
- "oid": true, "regclass": true, "regproc": true, "regtype": true, "regoper": true,
- "regoperator": true, "regconfig": true, "regdictionary": true,
- "tid": true, "xid": true, "cid": true,
- }
-
- return builtInTypes[baseType]
-}
-
-// extractColumnConstraint extracts column-level constraints
-func (e *metadataExtractor) extractColumnConstraint(ctx parser.IColconstraintelemContext, column *storepb.ColumnMetadata, table *storepb.TableMetadata, constraintName string, schemaName string) {
- if ctx == nil {
- return
- }
-
- switch {
- case ctx.NOT() != nil && ctx.NULL_P() != nil:
- column.Nullable = false
- case ctx.NULL_P() != nil && ctx.NOT() == nil:
- column.Nullable = true
- case ctx.DEFAULT() != nil:
- if expr := ctx.B_expr(); expr != nil {
- rawDefault := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)
- column.Default = e.normalizeDefaultValue(rawDefault, column, e.currentSchema)
- }
- case ctx.PRIMARY() != nil && ctx.KEY() != nil:
- column.Nullable = false
- // Create primary key index
- index := &storepb.IndexMetadata{
- Primary: true,
- Unique: true,
- IsConstraint: true,
- Expressions: []string{column.Name},
- Descending: []bool{false}, // Single column, ascending by default
- Type: "btree",
- Visible: false, // Match PostgreSQL database behavior
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
- // Use provided constraint name or generate one
- if constraintName != "" {
- index.Name = constraintName
- } else {
- index.Name = fmt.Sprintf("%s_pkey", table.Name)
- }
- // Generate definition for primary key index
- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)
- table.Indexes = append(table.Indexes, index)
- case ctx.UNIQUE() != nil:
- // Create unique index
- index := &storepb.IndexMetadata{
- Unique: true,
- IsConstraint: true,
- Expressions: []string{column.Name},
- Descending: []bool{false}, // Single column, ascending by default
- Type: "btree",
- Visible: false, // Match PostgreSQL database behavior
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
- // Use provided constraint name or generate one
- if constraintName != "" {
- index.Name = constraintName
- } else {
- index.Name = fmt.Sprintf("%s_%s_key", table.Name, column.Name)
- }
- // Generate definition for unique index
- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)
- table.Indexes = append(table.Indexes, index)
- case ctx.REFERENCES() != nil:
- // Foreign key constraint
- fk := &storepb.ForeignKeyMetadata{
- Columns: []string{column.Name},
- }
- if constraintName != "" {
- fk.Name = constraintName
- }
- if qualifiedName := ctx.Qualified_name(); qualifiedName != nil {
- refSchema, refTable := e.extractSchemaAndTableName(qualifiedName)
- fk.ReferencedSchema = refSchema
- fk.ReferencedTable = refTable
- }
- if optColumnList := ctx.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {
- fk.ReferencedColumns = extractColumnList(optColumnList.Columnlist())
- }
- // Extract ON DELETE/UPDATE actions
- fk.OnDelete = "NO ACTION" // Default
- fk.OnUpdate = "NO ACTION" // Default
- if keyActions := ctx.Key_actions(); keyActions != nil {
- if keyDelete := keyActions.Key_delete(); keyDelete != nil {
- fk.OnDelete = extractKeyAction(keyDelete)
- }
- if keyUpdate := keyActions.Key_update(); keyUpdate != nil {
- fk.OnUpdate = extractKeyActionUpdate(keyUpdate)
- }
- }
- // Extract MATCH type
- fk.MatchType = extractMatchType(ctx)
-
- // Generate constraint name if not provided (PostgreSQL auto-generates names)
- if fk.Name == "" {
- // For column-level foreign keys, use {table}_{column}_fkey format
- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, column.Name)
- }
- // TODO: Generate definition for foreign key (ForeignKeyMetadata might not have Definition field)
- // fk.Definition = e.generateForeignKeyDefinition(fk, table.Name, schemaName)
-
- if table.ForeignKeys == nil {
- table.ForeignKeys = []*storepb.ForeignKeyMetadata{}
- }
- table.ForeignKeys = append(table.ForeignKeys, fk)
- case ctx.GENERATED() != nil && ctx.IDENTITY_P() != nil:
- // Handle GENERATED ALWAYS AS IDENTITY or GENERATED BY DEFAULT AS IDENTITY
- if generatedWhen := ctx.Generated_when(); generatedWhen != nil {
- if generatedWhen.ALWAYS() != nil {
- column.IdentityGeneration = storepb.ColumnMetadata_ALWAYS
- } else if generatedWhen.BY() != nil && generatedWhen.DEFAULT() != nil {
- column.IdentityGeneration = storepb.ColumnMetadata_BY_DEFAULT
- }
- }
-
- // Create identity sequence for this column
- e.createIdentitySequence(table, column, schemaName)
- case ctx.CHECK() != nil:
- // Column-level CHECK constraint
- check := &storepb.CheckConstraintMetadata{}
-
- // Generate constraint name if not provided (PostgreSQL auto-generates names)
- if constraintName != "" {
- check.Name = constraintName
- } else {
- // For column-level check constraints, use {table}_{column}_check format
- check.Name = fmt.Sprintf("%s_%s_check", table.Name, column.Name)
- }
-
- // Extract the check expression
- if expr := ctx.A_expr(); expr != nil {
- rawExpression := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)
- check.Expression = strings.TrimSpace(rawExpression)
- }
-
- // Add to table's check constraints
- if table.CheckConstraints == nil {
- table.CheckConstraints = []*storepb.CheckConstraintMetadata{}
- }
- table.CheckConstraints = append(table.CheckConstraints, check)
- default:
- // Ignore other column constraints
- }
-}
-
-// extractTableConstraint extracts table-level constraints
-func (e *metadataExtractor) extractTableConstraint(ctx parser.ITableconstraintContext, table *storepb.TableMetadata, schemaName string) {
- if ctx == nil {
- return
- }
-
- constraintElem := ctx.Constraintelem()
- if constraintElem == nil {
- return
- }
-
- // Get constraint name
- var constraintName string
- if ctx.Name() != nil {
- constraintName = pgparser.NormalizePostgreSQLName(ctx.Name())
- }
-
- switch {
- case constraintElem.PRIMARY() != nil && constraintElem.KEY() != nil:
- // Primary key constraint
- index := &storepb.IndexMetadata{
- Name: constraintName,
- Primary: true,
- Unique: true,
- IsConstraint: true,
- Expressions: []string{},
- Descending: []bool{},
- Type: "btree",
- Visible: false, // Match PostgreSQL database behavior
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
- if index.Name == "" {
- index.Name = fmt.Sprintf("%s_pkey", table.Name)
- }
- // Extract columns
- // For PRIMARY KEY constraints, columns are in direct Columnlist, not Opt_column_list
- if columnList := constraintElem.Columnlist(); columnList != nil {
- columns := extractColumnList(columnList)
- index.Expressions = columns
- // Fill Descending array with false for all columns (default ascending)
- for range columns {
- index.Descending = append(index.Descending, false)
- }
- } else if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {
- // Fallback to Opt_column_list if direct Columnlist is not available
- columns := extractColumnList(optColumnList.Columnlist())
- index.Expressions = columns
- // Fill Descending array with false for all columns (default ascending)
- for range columns {
- index.Descending = append(index.Descending, false)
- }
- }
- // Generate definition for primary key constraint index
- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)
- table.Indexes = append(table.Indexes, index)
-
- case constraintElem.UNIQUE() != nil:
- // Unique constraint
- index := &storepb.IndexMetadata{
- Name: constraintName,
- Unique: true,
- IsConstraint: true,
- Expressions: []string{},
- Descending: []bool{},
- Type: "btree",
- Visible: false, // Match PostgreSQL database behavior
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
- // Extract columns
- // For UNIQUE constraints, columns are in direct Columnlist, not Opt_column_list
- if columnList := constraintElem.Columnlist(); columnList != nil {
- columns := extractColumnList(columnList)
- index.Expressions = columns
- // Fill Descending array with false for all columns (default ascending)
- for range columns {
- index.Descending = append(index.Descending, false)
- }
- } else if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {
- // Fallback to Opt_column_list if direct Columnlist is not available
- columns := extractColumnList(optColumnList.Columnlist())
- index.Expressions = columns
- // Fill Descending array with false for all columns (default ascending)
- for range columns {
- index.Descending = append(index.Descending, false)
- }
- }
- // Generate constraint name if not provided
- if index.Name == "" && len(index.Expressions) > 0 {
- // PostgreSQL automatically generates constraint names in the format: table_column_key
- // For multi-column constraints: table_column1_column2_..._key
- index.Name = fmt.Sprintf("%s_%s_key", table.Name, strings.Join(index.Expressions, "_"))
- }
- // Generate definition for unique constraint index
- index.Definition = e.generateConstraintIndexDefinition(index, table.Name, schemaName)
- table.Indexes = append(table.Indexes, index)
-
- case constraintElem.CHECK() != nil:
- // Check constraint
- check := &storepb.CheckConstraintMetadata{}
- // Generate constraint name if not provided (PostgreSQL auto-generates names)
- if constraintName != "" {
- check.Name = constraintName
- } else {
- // For table-level check constraints, use {table}_check_{index} format
- checkIndex := len(table.CheckConstraints) + 1
- check.Name = fmt.Sprintf("%s_check_%d", table.Name, checkIndex)
- }
- if expr := constraintElem.A_expr(); expr != nil {
- rawExpression := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(expr)
- check.Expression = strings.TrimSpace(rawExpression)
- }
- if table.CheckConstraints == nil {
- table.CheckConstraints = []*storepb.CheckConstraintMetadata{}
- }
- table.CheckConstraints = append(table.CheckConstraints, check)
-
- case constraintElem.FOREIGN() != nil && constraintElem.KEY() != nil:
- // Foreign key constraint
- fk := &storepb.ForeignKeyMetadata{
- Name: constraintName,
- Columns: []string{},
- ReferencedColumns: []string{},
- }
- // Extract local columns from Columnlist (before REFERENCES)
- if columnList := constraintElem.Columnlist(); columnList != nil {
- fk.Columns = extractColumnList(columnList)
- }
- // Extract referenced table
- if qualifiedName := constraintElem.Qualified_name(); qualifiedName != nil {
- refSchema, refTable := e.extractSchemaAndTableName(qualifiedName)
- fk.ReferencedSchema = refSchema
- fk.ReferencedTable = refTable
- }
- // Extract referenced columns from Opt_column_list (after REFERENCES table_name)
- if optColumnList := constraintElem.Opt_column_list(); optColumnList != nil && optColumnList.Columnlist() != nil {
- fk.ReferencedColumns = extractColumnList(optColumnList.Columnlist())
- } else if len(fk.Columns) > 0 {
- // If referenced columns not specified, assume they match the local columns
- fk.ReferencedColumns = make([]string, len(fk.Columns))
- copy(fk.ReferencedColumns, fk.Columns)
- }
- // Extract ON DELETE/UPDATE actions
- fk.OnDelete = "NO ACTION" // Default
- fk.OnUpdate = "NO ACTION" // Default
- if keyActions := constraintElem.Key_actions(); keyActions != nil {
- if keyDelete := keyActions.Key_delete(); keyDelete != nil {
- fk.OnDelete = extractKeyAction(keyDelete)
- }
- if keyUpdate := keyActions.Key_update(); keyUpdate != nil {
- fk.OnUpdate = extractKeyActionUpdate(keyUpdate)
- }
- }
- // Extract MATCH type
- fk.MatchType = extractMatchTypeFromConstraintElem(constraintElem)
-
- // Generate constraint name if not provided (PostgreSQL auto-generates names)
- if fk.Name == "" {
- // PostgreSQL typically uses {table}_{column}_fkey format for single column foreign keys
- if len(fk.Columns) == 1 {
- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, fk.Columns[0])
- } else if len(fk.Columns) > 1 {
- // For multi-column foreign keys, use first column name
- fk.Name = fmt.Sprintf("%s_%s_fkey", table.Name, fk.Columns[0])
- } else {
- // Fallback name
- fk.Name = fmt.Sprintf("%s_fkey", table.Name)
- }
- }
-
- if table.ForeignKeys == nil {
- table.ForeignKeys = []*storepb.ForeignKeyMetadata{}
- }
- table.ForeignKeys = append(table.ForeignKeys, fk)
-
- case constraintElem.EXCLUDE() != nil:
- // EXCLUDE constraint
- exclude := &storepb.ExcludeConstraintMetadata{}
- // Set constraint name
- if constraintName != "" {
- exclude.Name = constraintName
- } else {
- // Generate name if not provided (PostgreSQL auto-generates names)
- excludeIndex := len(table.ExcludeConstraints) + 1
- exclude.Name = fmt.Sprintf("%s_exclude_%d", table.Name, excludeIndex)
- }
-
- // Extract full EXCLUDE constraint expression from token stream
- // This includes "EXCLUDE USING gist (...)" format
- if parser := ctx.GetParser(); parser != nil {
- if tokenStream := parser.GetTokenStream(); tokenStream != nil {
- start := constraintElem.GetStart()
- stop := constraintElem.GetStop()
- if start != nil && stop != nil {
- rawExpression := tokenStream.GetTextFromTokens(start, stop)
- exclude.Expression = strings.TrimSpace(rawExpression)
- }
- }
- }
-
- if table.ExcludeConstraints == nil {
- table.ExcludeConstraints = []*storepb.ExcludeConstraintMetadata{}
- }
- table.ExcludeConstraints = append(table.ExcludeConstraints, exclude)
-
- default:
- // Other constraint types not handled
- }
-}
-
-// extractKeyAction extracts the foreign key delete action type
-func extractKeyAction(ctx parser.IKey_deleteContext) string {
- if ctx == nil {
- return "NO ACTION"
- }
- keyAction := ctx.Key_action()
- if keyAction == nil {
- return "NO ACTION"
- }
- switch {
- case keyAction.CASCADE() != nil:
- return "CASCADE"
- case keyAction.SET() != nil && keyAction.NULL_P() != nil:
- return "SET NULL"
- case keyAction.SET() != nil && keyAction.DEFAULT() != nil:
- return "SET DEFAULT"
- case keyAction.RESTRICT() != nil:
- return "RESTRICT"
- default:
- return "NO ACTION"
- }
-}
-
-// extractKeyActionUpdate extracts the foreign key update action type
-func extractKeyActionUpdate(ctx parser.IKey_updateContext) string {
- if ctx == nil {
- return "NO ACTION"
- }
- keyAction := ctx.Key_action()
- if keyAction == nil {
- return "NO ACTION"
- }
- switch {
- case keyAction.CASCADE() != nil:
- return "CASCADE"
- case keyAction.SET() != nil && keyAction.NULL_P() != nil:
- return "SET NULL"
- case keyAction.SET() != nil && keyAction.DEFAULT() != nil:
- return "SET DEFAULT"
- case keyAction.RESTRICT() != nil:
- return "RESTRICT"
- default:
- return "NO ACTION"
- }
-}
-
-// extractMatchType extracts the match type from column constraint context
-func extractMatchType(ctx parser.IColconstraintelemContext) string {
- if ctx == nil || ctx.Key_match() == nil {
- return "SIMPLE" // PostgreSQL default is MATCH SIMPLE
- }
- keyMatch := ctx.Key_match()
- switch {
- case keyMatch.FULL() != nil:
- return "FULL"
- case keyMatch.PARTIAL() != nil:
- return "PARTIAL"
- case keyMatch.SIMPLE() != nil:
- return "SIMPLE"
- default:
- return ""
- }
-}
-
-// extractMatchTypeFromConstraintElem extracts the match type from table constraint context
-func extractMatchTypeFromConstraintElem(ctx parser.IConstraintelemContext) string {
- if ctx == nil || ctx.Key_match() == nil {
- return "SIMPLE" // PostgreSQL default is MATCH SIMPLE
- }
- keyMatch := ctx.Key_match()
- switch {
- case keyMatch.FULL() != nil:
- return "FULL"
- case keyMatch.PARTIAL() != nil:
- return "PARTIAL"
- case keyMatch.SIMPLE() != nil:
- return "SIMPLE"
- default:
- return ""
- }
-}
-
-// extractColumnList extracts column names from columnlist
-func extractColumnList(ctx parser.IColumnlistContext) []string {
- if ctx == nil {
- return nil
- }
-
- var columns []string
- for _, colElem := range ctx.AllColumnElem() {
- if colElem.Colid() != nil {
- columns = append(columns, pgparser.NormalizePostgreSQLColid(colElem.Colid()))
- }
- }
- return columns
-}
-
-// extractPartitionExpression extracts the partition expression (e.g., "RANGE (sale_date)")
-func (*metadataExtractor) extractPartitionExpression(ctx parser.IOptpartitionspecContext) string {
- if ctx == nil {
- return ""
- }
-
- partitionSpec := ctx.Partitionspec()
- if partitionSpec == nil {
- return ""
- }
-
- // Extract the full partition expression including the method (e.g., "RANGE (sale_date)")
- expression := ctx.GetParser().GetTokenStream().GetTextFromTokens(partitionSpec.Colid().GetStart(), partitionSpec.CLOSE_PAREN().GetSymbol())
- return expression
-}
-
-// extractPartitionType extracts the partition type from the partition specification
-func (*metadataExtractor) extractPartitionType(ctx parser.IOptpartitionspecContext) storepb.TablePartitionMetadata_Type {
- if ctx == nil {
- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED
- }
-
- partitionSpec := ctx.Partitionspec()
- if partitionSpec == nil {
- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED
- }
-
- // Extract the partition method from the full text
- fullText := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(partitionSpec)
- upperText := strings.ToUpper(fullText)
-
- // Determine partition type from the method keyword after "PARTITION BY"
- if strings.Contains(upperText, "PARTITION BY RANGE") {
- return storepb.TablePartitionMetadata_RANGE
- } else if strings.Contains(upperText, "PARTITION BY LIST") {
- return storepb.TablePartitionMetadata_LIST
- } else if strings.Contains(upperText, "PARTITION BY HASH") {
- return storepb.TablePartitionMetadata_HASH
- }
-
- return storepb.TablePartitionMetadata_TYPE_UNSPECIFIED
-}
-
-// EnterAltertablestmt is called when entering an ALTER TABLE statement
-func (e *metadataExtractor) EnterAltertablestmt(ctx *parser.AltertablestmtContext) {
- if e.err != nil {
- return
- }
-
- // Check if this is an ATTACH PARTITION command
- if partitionCmd := ctx.Partition_cmd(); partitionCmd != nil {
- if partitionCmd.ATTACH() != nil && partitionCmd.PARTITION() != nil {
- e.handleAttachPartition(ctx, partitionCmd)
- }
- return
- }
-
- // Check if this is an ALTER TABLE command with alter_table_cmds
- if alterTableCmdList := ctx.Alter_table_cmds(); alterTableCmdList != nil {
- e.handleAlterTableCommands(ctx, alterTableCmdList)
- }
-}
-
-// handleAlterTableCommands processes ALTER TABLE commands like ADD CONSTRAINT
-func (e *metadataExtractor) handleAlterTableCommands(ctx *parser.AltertablestmtContext, alterTableCmdList parser.IAlter_table_cmdsContext) {
- // Get the table name from the ALTER TABLE statement
- if relationExpr := ctx.Relation_expr(); relationExpr != nil {
- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {
- schemaName, tableName := e.extractSchemaAndTableName(qualifiedName)
-
- // Get or create the table
- table := e.getOrCreateTable(schemaName, tableName)
-
- // Process each alter table command
- for _, alterTableCmd := range alterTableCmdList.AllAlter_table_cmd() {
- if alterTableCmd == nil {
- continue
- }
-
- // Debug: check what type of command this is
- hasAlter := alterTableCmd.ALTER() != nil
- hasAdd := alterTableCmd.ADD_P() != nil
- hasGenerated := alterTableCmd.GENERATED() != nil
- hasIdentity := alterTableCmd.IDENTITY_P() != nil
-
- // Check if this is an ADD CONSTRAINT command
- if alterTableCmd.ADD_P() != nil && alterTableCmd.Tableconstraint() != nil {
- e.extractTableConstraint(alterTableCmd.Tableconstraint(), table, schemaName)
- continue
- }
-
- // Check if this is an ALTER COLUMN ADD GENERATED AS IDENTITY command
- // According to g4: ALTER opt_column? colid ADD_P GENERATED generated_when AS IDENTITY_P
- // The COLUMN keyword is optional (opt_column?)
- if hasAlter && hasAdd && hasGenerated && hasIdentity {
- e.handleAlterColumn(alterTableCmd, table, schemaName)
- }
- }
- }
- }
-}
-
-// handleAlterColumn processes ALTER TABLE ALTER COLUMN commands
-func (e *metadataExtractor) handleAlterColumn(alterTableCmd parser.IAlter_table_cmdContext, table *storepb.TableMetadata, schemaName string) {
- // Get the column name from colid (based on g4: ALTER opt_column? colid ADD_P GENERATED...)
- columnName := ""
- allColids := alterTableCmd.AllColid()
- if len(allColids) > 0 {
- // The column name is the first colid
- columnName = pgparser.NormalizePostgreSQLColid(allColids[0])
- }
-
- // Debug log
- _ = columnName // TODO: remove after debug
-
- if columnName == "" {
- return
- }
-
- // Find the column in the table
- var targetColumn *storepb.ColumnMetadata
- for _, col := range table.Columns {
- if col.Name == columnName {
- targetColumn = col
- break
- }
- }
-
- // If column doesn't exist, we might need to create it
- // But for now, we'll just skip if the column doesn't exist
- if targetColumn == nil {
- return
- }
-
- // Check if this is ADD GENERATED AS IDENTITY (from g4: ALTER opt_column? colid ADD_P GENERATED generated_when AS IDENTITY_P)
- if alterTableCmd.ADD_P() != nil && alterTableCmd.GENERATED() != nil && alterTableCmd.IDENTITY_P() != nil {
- // Set identity generation mode
- if generatedWhen := alterTableCmd.Generated_when(); generatedWhen != nil {
- if generatedWhen.ALWAYS() != nil {
- targetColumn.IdentityGeneration = storepb.ColumnMetadata_ALWAYS
- } else if generatedWhen.BY() != nil && generatedWhen.DEFAULT() != nil {
- targetColumn.IdentityGeneration = storepb.ColumnMetadata_BY_DEFAULT
- }
- }
-
- // Get sequence options from AST
- var seqOptList parser.ISeqoptlistContext
- if optParenSeqList := alterTableCmd.Optparenthesizedseqoptlist(); optParenSeqList != nil {
- seqOptList = optParenSeqList.Seqoptlist()
- }
-
- // Create the identity sequence with options from AST
- e.createIdentitySequenceFromAST(table, targetColumn, schemaName, seqOptList)
- }
-}
-
-// handleAttachPartition processes ALTER TABLE ATTACH PARTITION statements
-func (e *metadataExtractor) handleAttachPartition(ctx *parser.AltertablestmtContext, partitionCmd parser.IPartition_cmdContext) {
- // Get the main table name
- if relationExpr := ctx.Relation_expr(); relationExpr != nil {
- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {
- mainSchema, mainTable := e.extractSchemaAndTableName(qualifiedName)
-
- // Get the partition name
- if partitionQualifiedName := partitionCmd.Qualified_name(); partitionQualifiedName != nil {
- partitionSchema, partitionName := e.extractSchemaAndTableName(partitionQualifiedName)
-
- // Get or create the main table
- mainTableMetadata := e.getOrCreateTable(mainSchema, mainTable)
-
- // Create partition metadata
- partition := &storepb.TablePartitionMetadata{
- Name: partitionName,
- }
-
- // Extract the FOR VALUES clause if present
- if partitionBound := partitionCmd.Partitionboundspec(); partitionBound != nil {
- partition.Value = e.extractPartitionBoundSpec(partitionBound)
- }
-
- // Add to main table's partitions
- if mainTableMetadata.Partitions == nil {
- mainTableMetadata.Partitions = []*storepb.TablePartitionMetadata{}
- }
- mainTableMetadata.Partitions = append(mainTableMetadata.Partitions, partition)
-
- // Ensure the partition table exists in our metadata
- // This handles cases where the partition table was created separately
- _ = e.getOrCreateTable(partitionSchema, partitionName)
- }
- }
- }
-}
-
-// EnterCreatematviewstmt is called when entering a CREATE MATERIALIZED VIEW statement
-func (e *metadataExtractor) EnterCreatematviewstmt(ctx *parser.CreatematviewstmtContext) {
- if e.err != nil {
- return
- }
-
- // Get the materialized view target
- mvTarget := ctx.Create_mv_target()
- if mvTarget == nil {
- return
- }
-
- // Extract schema and view name from the target
- var schemaName, viewName string
- if qualifiedName := mvTarget.Qualified_name(); qualifiedName != nil {
- schemaName, viewName = e.extractSchemaAndTableName(qualifiedName)
- } else {
- return
- }
-
- schema := e.getOrCreateSchema(schemaName)
-
- // Create materialized view metadata
- materializedView := &storepb.MaterializedViewMetadata{
- Name: viewName,
- }
-
- // Extract the view definition
- if ctx.AS() != nil && ctx.Selectstmt() != nil {
- // Get the full SELECT statement text
- selectCtx := ctx.Selectstmt()
- startToken := selectCtx.GetStart()
- stopToken := selectCtx.GetStop()
- if startToken != nil && stopToken != nil {
- materializedView.Definition = ctx.GetParser().GetTokenStream().GetTextFromTokens(startToken, stopToken)
- }
- }
-
- // Add to schema's materialized views
- if schema.MaterializedViews == nil {
- schema.MaterializedViews = []*storepb.MaterializedViewMetadata{}
- }
- schema.MaterializedViews = append(schema.MaterializedViews, materializedView)
-}
-
-// extractPartitionBoundSpec extracts the partition bound specification (FOR VALUES clause)
-func (*metadataExtractor) extractPartitionBoundSpec(ctx parser.IPartitionboundspecContext) string {
- if ctx == nil {
- return ""
- }
-
- // Get the full text of the partition bound specification
- return ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
-}
-
-// EnterCreateseqstmt is called when entering a create sequence statement
-func (e *metadataExtractor) EnterCreateseqstmt(ctx *parser.CreateseqstmtContext) {
- if e.err != nil {
- return
- }
-
- if ctx.Qualified_name() == nil {
- return
- }
-
- schemaName, sequenceName := e.extractSchemaAndTableName(ctx.Qualified_name())
- schemaMetadata := e.getOrCreateSchema(schemaName)
-
- sequence := &storepb.SequenceMetadata{
- Name: sequenceName,
- DataType: "bigint", // Default for PostgreSQL
- Start: "1",
- Increment: "1",
- MinValue: "1",
- MaxValue: "9223372036854775807",
- Cycle: false,
- CacheSize: "1",
- }
-
- // Extract sequence options
- if optSeqList := ctx.Optseqoptlist(); optSeqList != nil && optSeqList.Seqoptlist() != nil {
- for _, seqOptElem := range optSeqList.Seqoptlist().AllSeqoptelem() {
- if seqOptElem == nil {
- continue
- }
- e.extractSequenceOption(seqOptElem, sequence)
- }
- }
-
- // Store the sequence temporarily for OWNED BY processing
- e.sequences[fmt.Sprintf("%s.%s", schemaName, sequenceName)] = sequence
-
- if schemaMetadata.Sequences == nil {
- schemaMetadata.Sequences = []*storepb.SequenceMetadata{}
- }
- schemaMetadata.Sequences = append(schemaMetadata.Sequences, sequence)
-}
-
-// EnterAlterseqstmt is called when entering an ALTER SEQUENCE statement
-func (e *metadataExtractor) EnterAlterseqstmt(ctx *parser.AlterseqstmtContext) {
- if e.err != nil {
- return
- }
-
- if ctx.Qualified_name() == nil {
- return
- }
-
- schemaName, sequenceName := e.extractSchemaAndTableName(ctx.Qualified_name())
- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)
-
- // Find the existing sequence or create one if it doesn't exist
- sequence, exists := e.sequences[sequenceKey]
- if !exists {
- // If sequence doesn't exist yet, create it with defaults
- schemaMetadata := e.getOrCreateSchema(schemaName)
- sequence = &storepb.SequenceMetadata{
- Name: sequenceName,
- DataType: "bigint",
- Start: "1",
- Increment: "1",
- MinValue: "1",
- MaxValue: "9223372036854775807",
- Cycle: false,
- CacheSize: "1",
- }
- e.sequences[sequenceKey] = sequence
- if schemaMetadata.Sequences == nil {
- schemaMetadata.Sequences = []*storepb.SequenceMetadata{}
- }
- schemaMetadata.Sequences = append(schemaMetadata.Sequences, sequence)
- }
-
- // Process sequence options from ALTER SEQUENCE
- if seqOptList := ctx.Seqoptlist(); seqOptList != nil {
- for _, seqOptElem := range seqOptList.AllSeqoptelem() {
- if seqOptElem == nil {
- continue
- }
- e.extractSequenceOption(seqOptElem, sequence)
- }
- }
-}
-
-// EnterCommentstmt is called when entering a COMMENT statement
-func (e *metadataExtractor) EnterCommentstmt(ctx *parser.CommentstmtContext) {
- if e.err != nil {
- return
- }
-
- // Extract comment text (common for all comment types)
- var comment string
- if commentText := ctx.Comment_text(); commentText != nil {
- if commentText.NULL_P() != nil {
- // COMMENT ... IS NULL means remove comment
- comment = ""
- } else if commentText.Sconst() != nil {
- // Extract the string constant, removing quotes
- commentStr := commentText.GetText()
- if len(commentStr) >= 2 && commentStr[0] == '\'' && commentStr[len(commentStr)-1] == '\'' {
- // Remove quotes and unescape
- comment = commentStr[1 : len(commentStr)-1]
- // Unescape single quotes ('' -> ')
- comment = strings.ReplaceAll(comment, "''", "'")
- }
- }
- }
-
- // Handle COMMENT ON COLUMN
- if ctx.COLUMN() != nil && ctx.Any_name() != nil {
- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())
- if len(parts) < 2 {
- return
- }
-
- var schemaName, tableName, columnName string
- if len(parts) == 2 {
- schemaName = e.currentSchema
- tableName = parts[0]
- columnName = parts[1]
- } else if len(parts) == 3 {
- schemaName = parts[0]
- tableName = parts[1]
- columnName = parts[2]
- } else {
- return
- }
-
- schema := e.getOrCreateSchema(schemaName)
- for _, table := range schema.Tables {
- if table.Name == tableName {
- for i, col := range table.Columns {
- if col.Name == columnName {
- table.Columns[i].Comment = comment
- return
- }
- }
- }
- }
- return
- }
-
- // Handle COMMENT ON TABLE/SEQUENCE/VIEW etc (object_type_any_name)
- if ctx.Object_type_any_name() != nil && ctx.Any_name() != nil {
- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())
- if len(parts) == 0 {
- return
- }
-
- var schemaName, objectName string
- if len(parts) == 1 {
- schemaName = e.currentSchema
- objectName = parts[0]
- } else if len(parts) >= 2 {
- schemaName = parts[0]
- objectName = parts[1]
- }
-
- schema := e.getOrCreateSchema(schemaName)
- objectType := ctx.Object_type_any_name()
-
- // COMMENT ON TABLE
- if objectType.TABLE() != nil {
- for _, table := range schema.Tables {
- if table.Name == objectName {
- table.Comment = comment
- return
- }
- }
- }
-
- // COMMENT ON SEQUENCE
- if objectType.SEQUENCE() != nil {
- for _, seq := range schema.Sequences {
- if seq.Name == objectName {
- seq.Comment = comment
- return
- }
- }
- }
-
- // COMMENT ON VIEW
- if objectType.VIEW() != nil {
- for _, view := range schema.Views {
- if view.Name == objectName {
- view.Comment = comment
- return
- }
- }
- }
-
- // COMMENT ON MATERIALIZED VIEW
- if objectType.MATERIALIZED() != nil && objectType.VIEW() != nil {
- for _, view := range schema.MaterializedViews {
- if view.Name == objectName {
- view.Comment = comment
- return
- }
- }
- }
-
- // COMMENT ON INDEX
- if objectType.INDEX() != nil {
- for _, table := range schema.Tables {
- for i, index := range table.Indexes {
- if index.Name == objectName {
- table.Indexes[i].Comment = comment
- return
- }
- }
- }
- }
-
- return
- }
-
- // Handle COMMENT ON SCHEMA / EXTENSION / etc.
- if ctx.Object_type_name() != nil && ctx.Name() != nil {
- objectType := strings.ToUpper(ctx.Object_type_name().GetText())
- name := pgparser.NormalizePostgreSQLName(ctx.Name())
- switch objectType {
- case "SCHEMA":
- if name != "" {
- schema := e.getOrCreateSchema(name)
- schema.Comment = comment
- }
- case "EXTENSION":
- if ext, exists := e.extensions[name]; exists {
- ext.Description = comment
- }
- default:
- }
- return
- }
-
- // Handle COMMENT ON FUNCTION/PROCEDURE
- if (ctx.FUNCTION() != nil || ctx.PROCEDURE() != nil) && ctx.Function_with_argtypes() != nil {
- funcWithArgs := ctx.Function_with_argtypes()
- if funcWithArgs.Func_name() == nil {
- return
- }
-
- // Extract function name
- parts := pgparser.NormalizePostgreSQLFuncName(funcWithArgs.Func_name())
- if len(parts) == 0 {
- return
- }
-
- var schemaName, funcName string
- if len(parts) == 1 {
- schemaName = e.currentSchema
- funcName = parts[0]
- } else if len(parts) >= 2 {
- schemaName = parts[0]
- funcName = parts[1]
- }
-
- schema := e.getOrCreateSchema(schemaName)
- for i, function := range schema.Functions {
- if function.Name == funcName {
- schema.Functions[i].Comment = comment
- return
- }
- }
- return
- }
-
- // Handle COMMENT ON TYPE (including ENUM types)
- if ctx.TYPE_P() != nil && ctx.AllTypename() != nil && len(ctx.AllTypename()) > 0 {
- // Extract type name from typename
- typename := ctx.AllTypename()[0]
- var schemaName, typeName string
-
- // Try to get qualified name from typename
- if typename.Qualified_name() != nil {
- schemaName, typeName = e.extractSchemaAndTableName(typename.Qualified_name())
- } else if typename.Simpletypename() != nil {
- // Simple type name - get the text representation
- typeText := typename.GetParser().GetTokenStream().GetTextFromRuleContext(typename)
- // typeText might be "myschema.status" or just "status"
- // We need to split it if it contains a dot
- parts := strings.Split(typeText, ".")
- if len(parts) == 2 {
- schemaName = parts[0]
- typeName = parts[1]
- } else {
- typeName = typeText
- schemaName = e.currentSchema
- }
- }
-
- if typeName != "" {
- schema := e.getOrCreateSchema(schemaName)
- for i, enumType := range schema.EnumTypes {
- if enumType.Name == typeName {
- schema.EnumTypes[i].Comment = comment
- return
- }
- }
- }
- return
- }
-}
-
-// EnterViewstmt is called when entering a create view statement
-func (e *metadataExtractor) EnterViewstmt(ctx *parser.ViewstmtContext) {
- if e.err != nil {
- return
- }
-
- if ctx.Qualified_name() == nil {
- return
- }
-
- schemaName, viewName := e.extractSchemaAndTableName(ctx.Qualified_name())
- schemaMetadata := e.getOrCreateSchema(schemaName)
-
- // Create regular view metadata
- // Note: Materialized views are not currently supported by the parser and are handled by sync
- viewMetadata := &storepb.ViewMetadata{
- Name: viewName,
- }
-
- // Extract view definition
- if ctx.Selectstmt() != nil {
- viewMetadata.Definition = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Selectstmt())
- }
-
- if schemaMetadata.Views == nil {
- schemaMetadata.Views = []*storepb.ViewMetadata{}
- }
- schemaMetadata.Views = append(schemaMetadata.Views, viewMetadata)
-}
-
-// EnterCreatefunctionstmt is called when entering a create function statement
-func (e *metadataExtractor) EnterCreatefunctionstmt(ctx *parser.CreatefunctionstmtContext) {
- if e.err != nil {
- return
- }
-
- funcNameCtx := ctx.Func_name()
- if funcNameCtx == nil {
- return
- }
-
- // Use NormalizePostgreSQLFuncName which properly handles quoted identifiers
- parts := pgparser.NormalizePostgreSQLFuncName(funcNameCtx)
- if len(parts) == 0 {
- return
- }
-
- var schemaName, funcName string
- if len(parts) == 1 {
- schemaName = e.currentSchema
- funcName = parts[0]
- } else if len(parts) >= 2 {
- schemaName = parts[0]
- funcName = parts[1]
- }
-
- if funcName == "" {
- return
- }
-
- schemaMetadata := e.getOrCreateSchema(schemaName)
-
- functionMetadata := &storepb.FunctionMetadata{
- Name: funcName,
- Definition: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- Signature: ExtractFunctionSignature(ctx, funcName),
- }
-
- if schemaMetadata.Functions == nil {
- schemaMetadata.Functions = []*storepb.FunctionMetadata{}
- }
- schemaMetadata.Functions = append(schemaMetadata.Functions, functionMetadata)
-}
-
-// ExtractFunctionSignature extracts the function signature with parameter types
-func ExtractFunctionSignature(ctx *parser.CreatefunctionstmtContext, funcName string) string {
- var signature strings.Builder
- signature.WriteString(funcName)
- signature.WriteString(`(`)
-
- if funcArgs := ctx.Func_args_with_defaults(); funcArgs != nil {
- if funcArgsList := funcArgs.Func_args_with_defaults_list(); funcArgsList != nil {
- args := funcArgsList.AllFunc_arg_with_default()
- for i, arg := range args {
- if i > 0 {
- signature.WriteString(", ")
- }
-
- if funcArg := arg.Func_arg(); funcArg != nil {
- // Extract parameter name if present
- if paramName := funcArg.Param_name(); paramName != nil {
- // Param_name returns the parameter name directly as text
- signature.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(paramName))
- signature.WriteString(" ")
- }
-
- // Extract argument class (IN/OUT/INOUT/VARIADIC)
- if argClass := funcArg.Arg_class(); argClass != nil {
- if argClass.OUT_P() != nil {
- signature.WriteString("OUT ")
- } else if argClass.INOUT() != nil {
- signature.WriteString("INOUT ")
- } else if argClass.VARIADIC() != nil {
- signature.WriteString("VARIADIC ")
- }
- // IN is default and usually omitted
- }
-
- // Extract parameter type
- if funcType := funcArg.Func_type(); funcType != nil {
- if funcType.Typename() != nil {
- signature.WriteString(extractTypeName(funcType.Typename()))
- }
- }
- }
- }
- }
- }
-
- signature.WriteString(")")
- return signature.String()
-}
-
-// EnterCreatetrigstmt is called when entering a create trigger statement
-func (e *metadataExtractor) EnterCreatetrigstmt(ctx *parser.CreatetrigstmtContext) {
- if e.err != nil {
- return
- }
-
- // Extract trigger name using normalization to properly handle quoted identifiers
- triggerNameCtx := ctx.Name()
- if triggerNameCtx == nil {
- return
- }
-
- triggerName := pgparser.NormalizePostgreSQLName(triggerNameCtx)
- if triggerName == "" {
- return
- }
-
- // Extract table name that the trigger is on using normalization to properly handle quoted identifiers
- qualifiedNameCtx := ctx.Qualified_name()
- if qualifiedNameCtx == nil {
- return
- }
-
- // Use NormalizePostgreSQLQualifiedName to handle quotes correctly
- parts := pgparser.NormalizePostgreSQLQualifiedName(qualifiedNameCtx)
- var schemaName, tableName string
- if len(parts) == 1 {
- schemaName = e.currentSchema
- tableName = parts[0]
- } else if len(parts) == 2 {
- schemaName = parts[0]
- tableName = parts[1]
- }
-
- if tableName == "" {
- return
- }
-
- // Build trigger metadata
- triggerMetadata := &storepb.TriggerMetadata{
- Name: triggerName,
- Body: e.buildTriggerDefinition(ctx),
- }
-
- // Store trigger with table key
- tableKey := fmt.Sprintf("%s.%s", schemaName, tableName)
- if e.triggers[tableKey] == nil {
- e.triggers[tableKey] = []*storepb.TriggerMetadata{}
- }
- e.triggers[tableKey] = append(e.triggers[tableKey], triggerMetadata)
-}
-
-// buildTriggerDefinition builds the trigger definition from the CREATE TRIGGER context
-func (e *metadataExtractor) buildTriggerDefinition(ctx *parser.CreatetrigstmtContext) string {
- // Build trigger definition by extracting individual components
- // This is more reliable than trying to fix the concatenated text
-
- var parts []string
-
- // CREATE TRIGGER (uppercase to match system catalog)
- parts = append(parts, "CREATE TRIGGER")
-
- // Trigger name
- if nameCtx := ctx.Name(); nameCtx != nil {
- triggerName := strings.ToLower(nameCtx.GetText())
- parts = append(parts, triggerName)
- }
-
- // Timing (BEFORE/AFTER/INSTEAD OF) - uppercase
- if actionTimeCtx := ctx.Triggeractiontime(); actionTimeCtx != nil {
- timing := strings.ToUpper(actionTimeCtx.GetText())
- // Handle "INSTEADOF" -> "INSTEAD OF"
- if timing == "INSTEADOF" {
- timing = "INSTEAD OF"
- }
- parts = append(parts, timing)
- }
-
- // Events (INSERT/UPDATE/DELETE) - uppercase with OR separators
- if eventsCtx := ctx.Triggerevents(); eventsCtx != nil {
- events := strings.ToUpper(eventsCtx.GetText())
- // Parse and format events with proper OR separators
- events = e.formatTriggerEvents(events)
- parts = append(parts, events)
- }
-
- // ON table_name with schema qualification
- parts = append(parts, "ON")
- if qualifiedNameCtx := ctx.Qualified_name(); qualifiedNameCtx != nil {
- tableName := strings.ToLower(qualifiedNameCtx.GetText())
- // Add schema qualification if not present (sync always includes schema)
- if !strings.Contains(tableName, ".") {
- schemaName := "public"
- if e.currentSchema != "" {
- schemaName = e.currentSchema
- }
- tableName = schemaName + "." + tableName
- }
- parts = append(parts, tableName)
- }
-
- // FOR EACH ROW/STATEMENT - uppercase
- if forSpecCtx := ctx.Triggerforspec(); forSpecCtx != nil {
- forSpec := strings.ToUpper(forSpecCtx.GetText())
- // Handle "FOREACHROW" -> "FOR EACH ROW", etc.
- forSpec = e.formatTriggerForSpec(forSpec)
- parts = append(parts, forSpec)
- }
-
- // WHEN condition (optional) - uppercase
- if whenCtx := ctx.Triggerwhen(); whenCtx != nil {
- whenClause := strings.ToUpper(whenCtx.GetText())
- parts = append(parts, "WHEN", whenClause)
- }
-
- // EXECUTE FUNCTION/PROCEDURE - uppercase
- parts = append(parts, "EXECUTE")
-
- if funcOrProcCtx := ctx.Function_or_procedure(); funcOrProcCtx != nil {
- funcType := strings.ToUpper(funcOrProcCtx.GetText())
- parts = append(parts, funcType)
- } else {
- // Default to FUNCTION if not specified
- parts = append(parts, "FUNCTION")
- }
-
- // Function name with schema qualification
- if funcNameCtx := ctx.Func_name(); funcNameCtx != nil {
- funcName := strings.ToLower(funcNameCtx.GetText())
-
- // Add schema qualification if not present (sync always includes schema)
- if !strings.Contains(funcName, ".") {
- schemaName := "public"
- if e.currentSchema != "" {
- schemaName = e.currentSchema
- }
- funcName = schemaName + "." + funcName
- }
-
- // Function arguments
- funcCall := funcName + "("
- if funcArgsCtx := ctx.Triggerfuncargs(); funcArgsCtx != nil {
- args := strings.ToLower(funcArgsCtx.GetText())
- funcCall += args
- }
- funcCall += ")"
-
- parts = append(parts, funcCall)
- }
-
- return strings.Join(parts, " ")
-}
-
-// formatTriggerEvents formats trigger events with proper OR separators, preserving order from DDL
-func (*metadataExtractor) formatTriggerEvents(events string) string {
- // The system catalog returns events in lowercase, and the expected format seems to preserve
- // the order as written in the DDL. Let's normalize to match the expected lowercase format
-
- // Handle common concatenated patterns first
- eventsLower := strings.ToLower(events)
-
- // Direct mappings for common patterns to match expected system catalog format
- eventMap := map[string]string{
- "insertorupdateordelete": "insert or delete or update", // Expected format from test
- "insertordelete": "insert or delete",
- "insertorupdate": "insert or update",
- "updateordelete": "delete or update",
- "deleteorupdate": "delete or update",
- "updateorinsert": "insert or update",
- "deleteorinsert": "insert or delete",
- "update": "update",
- "delete": "delete",
- "insert": "insert",
- }
-
- if formatted, exists := eventMap[eventsLower]; exists {
- return strings.ToUpper(formatted)
- }
-
- // For compound events, try to split and normalize while preserving original order
- if strings.Contains(strings.ToUpper(events), "OR") {
- parts := strings.Split(strings.ToUpper(events), "OR")
- for i := range parts {
- parts[i] = strings.TrimSpace(parts[i])
- }
- return strings.Join(parts, " OR ")
- }
-
- // Return uppercase if no special handling needed
- return strings.ToUpper(events)
-}
-
-// formatTriggerForSpec formats trigger FOR EACH specifications
-func (*metadataExtractor) formatTriggerForSpec(forSpec string) string {
- specMap := map[string]string{
- "foreachrow": "FOR EACH ROW",
- "foreachstatement": "FOR EACH STATEMENT",
- "for each row": "FOR EACH ROW",
- "for each statement": "FOR EACH STATEMENT",
- }
-
- specLower := strings.ToLower(forSpec)
- if formatted, exists := specMap[specLower]; exists {
- return formatted
- }
-
- // Return uppercase if not found
- return strings.ToUpper(forSpec)
-}
-
-// EnterCreateextensionstmt is called when entering a create extension statement
-func (e *metadataExtractor) EnterCreateextensionstmt(ctx *parser.CreateextensionstmtContext) {
- if e.err != nil {
- return
- }
-
- if ctx.Name() == nil {
- return
- }
-
- extension := &storepb.ExtensionMetadata{
- Name: pgparser.NormalizePostgreSQLName(ctx.Name()),
- Schema: "public", // Default schema
- }
-
- extractExtensionOptions(extension, ctx.Create_extension_opt_list())
- e.extensions[extension.Name] = extension
-}
-
-// extractExtensionOptions populates Schema and Version from CREATE EXTENSION options.
-func extractExtensionOptions(extension *storepb.ExtensionMetadata, optList parser.ICreate_extension_opt_listContext) {
- if optList == nil {
- return
- }
- for _, optItem := range optList.AllCreate_extension_opt_item() {
- if optItem == nil {
- continue
- }
- if optItem.SCHEMA() != nil && optItem.Name() != nil {
- extension.Schema = pgparser.NormalizePostgreSQLName(optItem.Name())
- }
- if optItem.VERSION_P() != nil && optItem.Nonreservedword_or_sconst() != nil {
- extension.Version = stripSingleQuotes(optItem.Nonreservedword_or_sconst().GetText())
- }
- }
-}
-
-// stripSingleQuotes removes surrounding single quotes from a SQL string constant.
-func stripSingleQuotes(s string) string {
- if len(s) >= 2 && s[0] == '\'' && s[len(s)-1] == '\'' {
- return s[1 : len(s)-1]
- }
- return s
-}
-
-// EnterDefinestmt is called when entering a define statement (CREATE TYPE AS ENUM)
-func (e *metadataExtractor) EnterDefinestmt(ctx *parser.DefinestmtContext) {
- if e.err != nil {
- return
- }
-
- // Check if this is CREATE TYPE AS ENUM
- if ctx.CREATE() != nil && ctx.TYPE_P() != nil && ctx.AS() != nil && ctx.ENUM_P() != nil {
- // Extract type name
- typeNames := ctx.AllAny_name()
- if len(typeNames) == 0 {
- return
- }
-
- // Get the schema and enum name from the first Any_name (which should be the type name)
- typeName := typeNames[0]
- schemaName, enumName := e.extractSchemaAndEnumName(typeName)
- schemaMetadata := e.getOrCreateSchema(schemaName)
-
- // Create enum metadata
- enumType := &storepb.EnumTypeMetadata{
- Name: enumName,
- Values: []string{},
- }
-
- // Extract enum values
- if optEnumValList := ctx.Opt_enum_val_list(); optEnumValList != nil {
- if enumValList := optEnumValList.Enum_val_list(); enumValList != nil {
- for _, sconst := range enumValList.AllSconst() {
- if sconst != nil {
- value := extractStringConstant(sconst)
- if value != "" {
- enumType.Values = append(enumType.Values, value)
- }
- }
- }
- }
- }
-
- if schemaMetadata.EnumTypes == nil {
- schemaMetadata.EnumTypes = []*storepb.EnumTypeMetadata{}
- }
- schemaMetadata.EnumTypes = append(schemaMetadata.EnumTypes, enumType)
- }
-}
-
-// EnterIndexstmt is called when entering a create index statement
-func (e *metadataExtractor) EnterIndexstmt(ctx *parser.IndexstmtContext) {
- if e.err != nil {
- return
- }
-
- // Check if this is CREATE INDEX
- if ctx.CREATE() == nil || ctx.INDEX() == nil || ctx.ON() == nil {
- return
- }
-
- // Extract index name
- var indexName string
- if name := ctx.Name(); name != nil {
- indexName = pgparser.NormalizePostgreSQLName(name)
- }
-
- // If no explicit name, PostgreSQL will generate one - we can't predict it here
- if indexName == "" {
- return
- }
-
- // Extract table/view name from relation_expr
- if relationExpr := ctx.Relation_expr(); relationExpr != nil {
- if qualifiedName := relationExpr.Qualified_name(); qualifiedName != nil {
- schemaName, relationName := e.extractSchemaAndTableName(qualifiedName)
-
- // Try to find materialized view first, then table
- var indexTarget any
- var targetIndexes *[]*storepb.IndexMetadata
-
- if materializedView := e.findMaterializedView(schemaName, relationName); materializedView != nil {
- // Index is on a materialized view
- indexTarget = materializedView
- if materializedView.Indexes == nil {
- materializedView.Indexes = []*storepb.IndexMetadata{}
- }
- targetIndexes = &materializedView.Indexes
- } else {
- // Index is on a table - get or create the table
- tableMetadata := e.getOrCreateTable(schemaName, relationName)
- indexTarget = tableMetadata
- targetIndexes = &tableMetadata.Indexes
- }
-
- // If we couldn't find either table or materialized view, skip
- if indexTarget == nil || targetIndexes == nil {
- return
- }
-
- // Create index metadata
- index := &storepb.IndexMetadata{
- Name: indexName,
- Expressions: []string{},
- Descending: []bool{},
- Unique: false,
- Primary: false,
- IsConstraint: false,
- Visible: false, // Match PostgreSQL database behavior
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
-
- // Check if it's a unique index
- if optUnique := ctx.Opt_unique(); optUnique != nil {
- index.Unique = true
- }
-
- // Extract index method (BTREE, HASH, GIN, GIST, etc.)
- // Default to btree if not specified
- index.Type = "btree"
- if accessMethod := ctx.Access_method_clause(); accessMethod != nil {
- if accessMethod.USING() != nil && accessMethod.Name() != nil {
- index.Type = strings.ToLower(pgparser.NormalizePostgreSQLName(accessMethod.Name()))
- }
- }
-
- // Extract index parameters (columns/expressions)
- if indexParams := ctx.Index_params(); indexParams != nil {
- for _, indexElem := range indexParams.AllIndex_elem() {
- if indexElem == nil {
- continue
- }
-
- // Extract column name or expression
- var expression string
- if colid := indexElem.Colid(); colid != nil {
- // Simple column reference
- expression = pgparser.NormalizePostgreSQLColid(colid)
- } else if funcExpr := indexElem.Func_expr_windowless(); funcExpr != nil {
- // Function expression - parse semantically for better compatibility
- expression = e.extractFunctionExpression(ctx, funcExpr)
- } else if aExpr := indexElem.A_expr(); aExpr != nil {
- // General expression (in parentheses)
- expression = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(aExpr)
- }
-
- if expression != "" {
- index.Expressions = append(index.Expressions, expression)
-
- // Extract sort order (ASC/DESC) from index element options
- isDescending := false
- if options := indexElem.Index_elem_options(); options != nil {
- if ascDesc := options.Opt_asc_desc(); ascDesc != nil {
- if ascDesc.DESC() != nil {
- isDescending = true
- }
- // ASC is default, so we don't need to check for it explicitly
- }
- }
- index.Descending = append(index.Descending, isDescending)
- // Don't set KeyLength - PostgreSQL database doesn't return this information
- }
- }
- }
-
- // Generate definition for the index
- index.Definition = e.generateIndexDefinition(ctx, index, schemaName, relationName)
-
- // Add the index to the table or materialized view
- *targetIndexes = append(*targetIndexes, index)
- }
- }
-}
-
-// generateIndexDefinition generates the CREATE INDEX definition for an index
-func (e *metadataExtractor) generateIndexDefinition(ctx *parser.IndexstmtContext, index *storepb.IndexMetadata, schemaName, tableName string) string {
- var parts []string
-
- // Start with CREATE [UNIQUE] INDEX
- if index.Unique {
- parts = append(parts, "CREATE UNIQUE INDEX")
- } else {
- parts = append(parts, "CREATE INDEX")
- }
-
- // Add index name
- parts = append(parts, index.Name)
-
- // Add ON table
- parts = append(parts, "ON")
- if schemaName != "" && schemaName != "public" {
- parts = append(parts, fmt.Sprintf("%s.%s", schemaName, tableName))
- } else {
- parts = append(parts, fmt.Sprintf("public.%s", tableName))
- }
-
- // Add USING method (default is btree)
- if index.Type != "" && index.Type != "btree" {
- parts = append(parts, "USING", index.Type)
- } else {
- parts = append(parts, "USING btree")
- }
-
- // Add column list
- if len(index.Expressions) > 0 {
- columnList := make([]string, len(index.Expressions))
- for i, expr := range index.Expressions {
- quoted := quoteIndexExpression(expr)
- // Add DESC if needed
- if i < len(index.Descending) && index.Descending[i] {
- columnList[i] = fmt.Sprintf("%s DESC", quoted)
- } else {
- columnList[i] = quoted
- }
- }
- parts = append(parts, fmt.Sprintf("(%s)", strings.Join(columnList, ", ")))
- }
-
- // Check for INCLUDE clause (covering index)
- fullStatement := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- if includeClause := e.extractIncludeClause(fullStatement); includeClause != "" {
- parts = append(parts, includeClause)
- }
-
- // Check for WHERE clause (partial index)
- if ctx.Where_clause() != nil {
- whereClause := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())
- // Add WHERE clause as-is
- if whereClause != "" {
- parts = append(parts, whereClause)
- }
- }
-
- // End with semicolon
- return strings.Join(parts, " ") + ";"
-}
-
-// extractIncludeClause extracts the INCLUDE clause from a CREATE INDEX statement
-func (*metadataExtractor) extractIncludeClause(statement string) string {
- // Convert to lowercase for case-insensitive matching
- lowerStatement := strings.ToLower(statement)
-
- // Find the INCLUDE keyword
- includeIdx := strings.Index(lowerStatement, " include ")
- if includeIdx == -1 {
- return ""
- }
-
- // Find the opening parenthesis after INCLUDE
- searchStart := includeIdx + 9 // length of " include "
- openParenIdx := strings.Index(lowerStatement[searchStart:], "(")
- if openParenIdx == -1 {
- return ""
- }
- openParenIdx += searchStart
-
- // Find the matching closing parenthesis
- parenCount := 1
- i := openParenIdx + 1
- for i < len(lowerStatement) && parenCount > 0 {
- switch lowerStatement[i] {
- case '(':
- parenCount++
- case ')':
- parenCount--
- default:
- // Other characters don't affect parentheses counting
- }
- i++
- }
-
- if parenCount > 0 {
- return "" // Unmatched parentheses
- }
-
- // Extract the INCLUDE clause from the original statement (preserving case)
- includeClause := statement[includeIdx+1 : i] // +1 to skip the leading space
- return strings.TrimSpace(includeClause)
-}
-
-// generateConstraintIndexDefinition generates the CREATE INDEX definition for constraint-based indexes
-func (*metadataExtractor) generateConstraintIndexDefinition(index *storepb.IndexMetadata, tableName, schemaName string) string {
- var parts []string
-
- // Start with CREATE [UNIQUE] INDEX
- if index.Unique {
- parts = append(parts, "CREATE UNIQUE INDEX")
- } else {
- parts = append(parts, "CREATE INDEX")
- }
-
- // Add index name
- parts = append(parts, index.Name)
-
- // Add ON table
- parts = append(parts, "ON")
- if schemaName != "" && schemaName != "public" {
- parts = append(parts, fmt.Sprintf("%s.%s", schemaName, tableName))
- } else {
- parts = append(parts, fmt.Sprintf("public.%s", tableName))
- }
-
- // Add USING method (default is btree)
- if index.Type != "" && index.Type != "btree" {
- parts = append(parts, "USING", index.Type)
- } else {
- parts = append(parts, "USING btree")
- }
-
- // Add column list
- if len(index.Expressions) > 0 {
- columnList := make([]string, len(index.Expressions))
- for i, expr := range index.Expressions {
- quoted := quoteIndexExpression(expr)
- // Add DESC if needed
- if i < len(index.Descending) && index.Descending[i] {
- columnList[i] = fmt.Sprintf("%s DESC", quoted)
- } else {
- columnList[i] = quoted
- }
- }
- parts = append(parts, fmt.Sprintf("(%s)", strings.Join(columnList, ", ")))
- }
- // End with semicolon
- return strings.Join(parts, " ") + ";"
-}
-
-// extractSequenceOption extracts sequence options
-func (*metadataExtractor) extractSequenceOption(ctx parser.ISeqoptelemContext, sequence *storepb.SequenceMetadata) {
- if ctx == nil || sequence == nil {
- return
- }
-
- switch {
- case ctx.AS() != nil && ctx.Simpletypename() != nil:
- // Data type - preserve the original case
- sequence.DataType = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Simpletypename())
- case ctx.INCREMENT() != nil && ctx.Numericonly() != nil:
- // INCREMENT BY
- sequence.Increment = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())
- case ctx.MINVALUE() != nil && ctx.Numericonly() != nil:
- // MINVALUE
- sequence.MinValue = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())
- case ctx.MAXVALUE() != nil && ctx.Numericonly() != nil:
- // MAXVALUE
- sequence.MaxValue = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())
- case ctx.START() != nil && ctx.Numericonly() != nil:
- // START WITH
- sequence.Start = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())
- case ctx.CACHE() != nil && ctx.Numericonly() != nil:
- // CACHE
- sequence.CacheSize = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Numericonly())
- case ctx.CYCLE() != nil:
- // CYCLE
- sequence.Cycle = true
- case ctx.NO() != nil && ctx.CYCLE() != nil:
- // NO CYCLE
- sequence.Cycle = false
- case ctx.OWNED() != nil && ctx.BY() != nil && ctx.Any_name() != nil:
- // OWNED BY table.column
- parts := pgparser.NormalizePostgreSQLAnyName(ctx.Any_name())
- if len(parts) == 2 {
- // Format: table.column
- sequence.OwnerTable = parts[0]
- sequence.OwnerColumn = parts[1]
- } else if len(parts) == 3 {
- // Format: schema.table.column (we only care about table.column)
- sequence.OwnerTable = parts[1]
- sequence.OwnerColumn = parts[2]
- }
- default:
- // Other sequence options
- }
-}
-
-// createIdentitySequence creates an identity sequence for a column with GENERATED AS IDENTITY
-func (e *metadataExtractor) createIdentitySequence(table *storepb.TableMetadata, column *storepb.ColumnMetadata, schemaName string) {
- // Create identity sequence name following PostgreSQL conventions
- // Format: {table_name}_{column_name}_seq
- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)
-
- // Determine sequence data type and limits based on column type
- // For identity columns, use positive ranges starting from 1
- var dataType, minValue, maxValue string
- switch strings.ToUpper(column.Type) {
- case "SMALLINT", "INT2":
- dataType = "smallint"
- minValue = "1"
- maxValue = "32767"
- case "INTEGER", "INT", "INT4":
- dataType = "integer"
- minValue = "1"
- maxValue = "2147483647"
- case "BIGINT", "INT8":
- dataType = "bigint"
- minValue = "1"
- maxValue = "9223372036854775807"
- default:
- // Default to bigint for unknown types
- dataType = "bigint"
- minValue = "1"
- maxValue = "9223372036854775807"
- }
-
- // Create the sequence metadata
- sequence := &storepb.SequenceMetadata{
- Name: sequenceName,
- DataType: dataType,
- Start: "1",
- Increment: "1",
- MinValue: minValue,
- MaxValue: maxValue,
- Cycle: false,
- CacheSize: "1",
- OwnerTable: table.Name,
- OwnerColumn: column.Name,
- }
-
- // Add the sequence to the schema
- schema := e.getOrCreateSchema(schemaName)
- if schema.Sequences == nil {
- schema.Sequences = []*storepb.SequenceMetadata{}
- }
- schema.Sequences = append(schema.Sequences, sequence)
-
- // Store in the sequences map for reference
- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)
- e.sequences[sequenceKey] = sequence
-}
-
-// createIdentitySequenceFromAST creates an identity sequence from ALTER COLUMN ADD IDENTITY using AST
-func (e *metadataExtractor) createIdentitySequenceFromAST(table *storepb.TableMetadata, column *storepb.ColumnMetadata, schemaName string, seqOptList parser.ISeqoptlistContext) {
- // Default sequence name
- sequenceName := fmt.Sprintf("%s_%s_seq", table.Name, column.Name)
-
- // Determine sequence data type and limits based on column type
- // For identity columns, use positive ranges starting from 1
- var dataType, minValue, maxValue string
- switch strings.ToUpper(column.Type) {
- case "SMALLINT", "INT2":
- dataType = "smallint"
- minValue = "1"
- maxValue = "32767"
- case "INTEGER", "INT", "INT4":
- dataType = "integer"
- minValue = "1"
- maxValue = "2147483647"
- case "BIGINT", "INT8":
- dataType = "bigint"
- minValue = "1"
- maxValue = "9223372036854775807"
- default:
- // Default to bigint for unknown types
- dataType = "bigint"
- minValue = "1"
- maxValue = "9223372036854775807"
- }
-
- // Create the sequence metadata with defaults
- sequence := &storepb.SequenceMetadata{
- Name: sequenceName,
- DataType: dataType,
- Start: "1",
- Increment: "1",
- MinValue: minValue,
- MaxValue: maxValue,
- Cycle: false,
- CacheSize: "1",
- OwnerTable: table.Name,
- OwnerColumn: column.Name,
- }
-
- // Override with specified options from ALTER COLUMN statement (from AST)
- if seqOptList != nil {
- for _, seqElem := range seqOptList.AllSeqoptelem() {
- if seqElem == nil {
- continue
- }
- // Check for SEQUENCE NAME option to override default name
- if seqElem.SEQUENCE() != nil && seqElem.NAME_P() != nil {
- if anyName := seqElem.Any_name(); anyName != nil {
- _, seqName := e.extractSchemaAndEnumName(anyName)
- if seqName != "" {
- sequenceName = seqName
- sequence.Name = sequenceName
- }
- }
- }
- // Extract other sequence options
- e.extractSequenceOption(seqElem, sequence)
- }
- }
-
- // Add the sequence to the schema
- schema := e.getOrCreateSchema(schemaName)
- if schema.Sequences == nil {
- schema.Sequences = []*storepb.SequenceMetadata{}
- }
- schema.Sequences = append(schema.Sequences, sequence)
-
- // Store in the sequences map for reference
- sequenceKey := fmt.Sprintf("%s.%s", schemaName, sequenceName)
- e.sequences[sequenceKey] = sequence
-}
-
-// extractSchemaAndEnumName extracts schema and enum name from Any_name context
-func (e *metadataExtractor) extractSchemaAndEnumName(ctx parser.IAny_nameContext) (string, string) {
- if ctx == nil {
- return e.currentSchema, ""
- }
-
- parts := pgparser.NormalizePostgreSQLAnyName(ctx)
- if len(parts) == 1 {
- return e.currentSchema, parts[0]
- } else if len(parts) == 2 {
- return parts[0], parts[1]
- }
- return e.currentSchema, ""
-}
-
-// extractStringConstant extracts string value from Sconst context
-func extractStringConstant(ctx parser.ISconstContext) string {
- if ctx == nil {
- return ""
- }
- // Get the text and remove surrounding quotes
- text := ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- if len(text) >= 2 && text[0] == '\'' && text[len(text)-1] == '\'' {
- // Remove surrounding single quotes and handle escaped quotes
- result := text[1 : len(text)-1]
- result = strings.ReplaceAll(result, "''", "'")
- return result
- }
- return text
-}
-
-// normalizeDefaultValue normalizes default values to match PostgreSQL's stored format
-func (*metadataExtractor) normalizeDefaultValue(rawDefault string, column *storepb.ColumnMetadata, schemaName string) string {
- if rawDefault == "" {
- return ""
- }
-
- // Handle nextval() for sequences - add schema and regclass cast
- if strings.Contains(rawDefault, "nextval(") {
- // Pattern: nextval('sequence_name') -> nextval('schema.sequence_name'::regclass)
- // Extract sequence name from nextval('sequence_name')
- if start := strings.Index(rawDefault, "nextval('"); start != -1 {
- start += len("nextval('")
- if end := strings.Index(rawDefault[start:], "'"); end != -1 {
- sequenceName := rawDefault[start : start+end]
-
- // If sequence name doesn't have schema prefix, add current schema
- if !strings.Contains(sequenceName, ".") {
- return fmt.Sprintf("nextval('%s.%s'::regclass)", schemaName, sequenceName)
- }
- return fmt.Sprintf("nextval('%s'::regclass)", sequenceName)
- }
- }
- }
-
- // Handle ENUM default values - add type cast
- if column.Type != "" && strings.Contains(column.Type, ".") {
- // If column type is schema-qualified (e.g., "public.status_enum")
- // and default is a string literal, add type cast
- if strings.HasPrefix(rawDefault, "'") && strings.HasSuffix(rawDefault, "'") {
- return fmt.Sprintf("%s::%s", rawDefault, column.Type)
- }
- }
-
- return rawDefault
-}
-
-// extractFunctionExpression extracts function expressions preserving the original user input.
-func (*metadataExtractor) extractFunctionExpression(ctx *parser.IndexstmtContext, funcExpr parser.IFunc_expr_windowlessContext) string {
- if funcExpr == nil {
- return ""
- }
-
- // Return the original text without any normalization
- // This preserves the user's original input format
- return ctx.GetParser().GetTokenStream().GetTextFromRuleContext(funcExpr)
-}
-
-// extractViewDependencies analyzes view definitions to extract dependencies using GetQuerySpan
-func extractViewDependencies(schemaMetadata *storepb.DatabaseSchemaMetadata) {
- // Extract dependencies for each view
- for _, schema := range schemaMetadata.Schemas {
- for _, view := range schema.Views {
- view.DependencyColumns = getViewDependencies(view.Definition, schema.Name, schemaMetadata)
- }
-
- for _, mv := range schema.MaterializedViews {
- mv.DependencyColumns = getViewDependencies(mv.Definition, schema.Name, schemaMetadata)
- }
- }
-}
-
-// getViewDependencies extracts the tables/views that a view depends on using GetQuerySpan
-func getViewDependencies(viewDef string, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {
- // viewDef is already a SELECT statement extracted from the parsed CREATE VIEW statement
- queryStatement := strings.TrimSpace(viewDef)
-
- // Use GetQuerySpan with the full schema metadata so it can resolve table/view references
- span, err := pgparser.GetQuerySpan(
- context.Background(),
- base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {
- // Return the full schema metadata so GetQuerySpan can resolve references
- dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- return databaseName, dbMetadata, nil
- },
- ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {
- // Return empty list - we don't need actual database names for dependency extraction
- return []string{}, nil
- },
- },
- base.Statement{Text: queryStatement},
- "", // database
- schemaName,
- false, // case sensitive
- )
-
- // If error parsing query span, return empty dependencies
- if err != nil {
- return []*storepb.DependencyColumn{} // nolint:nilerr
- }
-
- // Collect unique dependencies
- dependencyMap := make(map[string]*storepb.DependencyColumn)
- for sourceColumn := range span.SourceColumns {
- // Create dependency key in format: schema.table
- key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)
- if _, exists := dependencyMap[key]; !exists {
- dependencyMap[key] = &storepb.DependencyColumn{
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- Column: "*", // Use wildcard since we're tracking table-level dependencies
- }
- }
- }
-
- // Convert map to slice
- var dependencies []*storepb.DependencyColumn
- for _, dep := range dependencyMap {
- dependencies = append(dependencies, dep)
- }
-
- return dependencies
-}
-
-// TODO: Add support for more PostgreSQL constructs if needed
-// (e.g., triggers, materialized views, custom types, etc.)
diff --git a/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.go b/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.go
deleted file mode 100644
index 3ca50ec23b..0000000000
--- a/backend/plugin/schema/pg/get_database_metadata_testcontainer_test.go
+++ /dev/null
@@ -1,2318 +0,0 @@
-package pg
-
-import (
- "context"
- "fmt"
- "strings"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/schema"
- "github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"
- "github.com/bytebase/bytebase/backend/store/model"
-)
-
-// TestGetDatabaseMetadataWithTestcontainer tests the get_database_metadata function
-// by comparing its output with the metadata retrieved from a real PostgreSQL instance.
-//
-//nolint:tparallel
-func TestGetDatabaseMetadataWithTestcontainer(t *testing.T) {
- ctx := context.Background()
-
- // Start shared PostgreSQL container for all subtests
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- // Test cases with various PostgreSQL features
- testCases := []struct {
- name string
- ddl string
- }{
- {
- name: "bytebase_schema",
- ddl: `
- CREATE TABLE employee (
- emp_no SERIAL NOT NULL,
- birth_date DATE NOT NULL,
- first_name TEXT NOT NULL,
- last_name TEXT NOT NULL,
- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,
- hire_date DATE NOT NULL,
- PRIMARY KEY (emp_no)
-);
-
-CREATE INDEX idx_employee_hire_date ON employee (hire_date);
-
-CREATE TABLE department (
- dept_no TEXT NOT NULL,
- dept_name TEXT NOT NULL,
- PRIMARY KEY (dept_no),
- UNIQUE (dept_name)
-);
-
-CREATE TABLE dept_manager (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE dept_emp (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE title (
- emp_no INT NOT NULL,
- title TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, title, from_date)
-);
-
-CREATE TABLE salary (
- emp_no INT NOT NULL,
- amount INT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, from_date)
-);
-
-CREATE INDEX idx_salary_amount ON salary (amount);
-
-CREATE TABLE audit (
- id SERIAL PRIMARY KEY,
- operation TEXT NOT NULL,
- query TEXT,
- user_name TEXT NOT NULL,
- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_audit_operation ON audit (operation);
-CREATE INDEX idx_audit_username ON audit (user_name);
-CREATE INDEX idx_audit_changed_at ON audit (changed_at);
-
-CREATE OR REPLACE FUNCTION log_dml_operations() RETURNS TRIGGER AS $$
-BEGIN
- IF (TG_OP = 'INSERT') THEN
- INSERT INTO audit (operation, query, user_name)
- VALUES ('INSERT', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'UPDATE') THEN
- INSERT INTO audit (operation, query, user_name)
- VALUES ('UPDATE', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'DELETE') THEN
- INSERT INTO audit (operation, query, user_name)
- VALUES ('DELETE', current_query(), current_user);
- RETURN OLD;
- END IF;
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
--- only log update and delete, otherwise, it will cause too much change.
-CREATE TRIGGER salary_log_trigger
-AFTER UPDATE OR DELETE ON salary
-FOR EACH ROW
-EXECUTE FUNCTION log_dml_operations();
-
-CREATE OR REPLACE VIEW dept_emp_latest_date AS
-SELECT
- emp_no,
- MAX(
- from_date) AS from_date,
- MAX(
- to_date) AS to_date
-FROM
- dept_emp
-GROUP BY
- emp_no;
-
--- shows only the current department for each employee
-CREATE OR REPLACE VIEW current_dept_emp AS
-SELECT
- l.emp_no,
- dept_no,
- l.from_date,
- l.to_date
-FROM
- dept_emp d
- INNER JOIN dept_emp_latest_date l ON d.emp_no = l.emp_no
- AND d.from_date = l.from_date
- AND l.to_date = d.to_date;
- `,
- },
- {
- name: "basic_tables_with_constraints",
- ddl: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- updated_at TIMESTAMP
-);
-
-CREATE TABLE posts (
- id SERIAL PRIMARY KEY,
- user_id INTEGER NOT NULL,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- published BOOLEAN DEFAULT false,
- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
-);
-
-CREATE INDEX idx_posts_user_id ON posts(user_id);
-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;
-`,
- },
- {
- name: "sequences_and_custom_types",
- ddl: `
-CREATE TYPE status_enum AS ENUM ('pending', 'active', 'inactive', 'deleted');
-CREATE TYPE mood AS ENUM ('happy', 'sad', 'neutral');
-
-CREATE SEQUENCE custom_id_seq START WITH 1000 INCREMENT BY 10;
-
-CREATE TABLE items (
- id INTEGER DEFAULT nextval('custom_id_seq') PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- status status_enum DEFAULT 'pending',
- user_mood mood
-);
-`,
- },
- {
- name: "views_and_functions",
- ddl: `
-CREATE TABLE employees (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- department VARCHAR(50),
- salary DECIMAL(10, 2)
-);
-
-CREATE VIEW active_employees AS
-SELECT id,
- name,
- department
- FROM employees
-WHERE (department IS NOT NULL);
-
-CREATE FUNCTION get_employee_count(dept VARCHAR) RETURNS INTEGER AS $$
-BEGIN
- RETURN (SELECT COUNT(*) FROM employees WHERE department = dept);
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE FUNCTION calculate_bonus(emp_id INTEGER) RETURNS DECIMAL AS $$
-DECLARE
- emp_salary DECIMAL;
-BEGIN
- SELECT salary INTO emp_salary FROM employees WHERE id = emp_id;
- RETURN emp_salary * 0.1;
-END;
-$$ LANGUAGE plpgsql;
-`,
- },
- {
- name: "partitioned_tables",
- ddl: `
-CREATE TABLE sales (
- id SERIAL,
- sale_date DATE NOT NULL,
- amount DECIMAL(10, 2) NOT NULL,
- region VARCHAR(50) NOT NULL
-) PARTITION BY RANGE (sale_date);
-
-CREATE TABLE sales_2023_q1 PARTITION OF sales
-FOR VALUES FROM ('2023-01-01') TO ('2023-04-01');
-
-CREATE TABLE sales_2023_q2 PARTITION OF sales
-FOR VALUES FROM ('2023-04-01') TO ('2023-07-01');
-
-CREATE TABLE sales_2023_q3 PARTITION OF sales
-FOR VALUES FROM ('2023-07-01') TO ('2023-10-01');
-
-CREATE TABLE sales_2023_q4 PARTITION OF sales
-FOR VALUES FROM ('2023-10-01') TO ('2024-01-01');
-`,
- },
- {
- name: "extensions_and_advanced_features",
- ddl: `
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
-
-CREATE TABLE documents (
- id UUID DEFAULT uuid_generate_v4() PRIMARY KEY,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- tags TEXT[],
- metadata JSONB,
- created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_documents_metadata ON documents USING GIN (metadata);
-CREATE INDEX idx_documents_tags ON documents USING GIN (tags);
-`,
- },
- {
- name: "indexes_with_asc_desc",
- ddl: `
-CREATE TABLE orders (
- id SERIAL PRIMARY KEY,
- customer_id INTEGER NOT NULL,
- order_date DATE NOT NULL,
- total_amount DECIMAL(10, 2) NOT NULL,
- status VARCHAR(20)
-);
-
--- Index with explicit DESC
-CREATE INDEX idx_orders_date_desc ON orders(order_date DESC);
-
--- Index with multiple columns, mixed ASC/DESC
-CREATE INDEX idx_orders_customer_date ON orders(customer_id ASC, order_date DESC);
-
--- Index with expressions and DESC
-CREATE INDEX idx_orders_year_month ON orders(EXTRACT( year FROM order_date ) DESC, EXTRACT( month FROM order_date ) ASC);
-
--- Unique index with DESC
-CREATE UNIQUE INDEX idx_orders_customer_status ON orders(customer_id, status DESC) WHERE status IS NOT NULL;
-`,
- },
- {
- name: "materialized_views_and_triggers",
- ddl: `
-CREATE TABLE audit_log (
- id SERIAL PRIMARY KEY,
- table_name VARCHAR(50) NOT NULL,
- operation VARCHAR(10) NOT NULL,
- user_id INTEGER,
- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- old_values JSONB,
- new_values JSONB
-);
-
-CREATE TABLE users_mv (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL,
- email VARCHAR(100) NOT NULL,
- last_login TIMESTAMP,
- login_count INTEGER DEFAULT 0
-);
-
--- Materialized view for user statistics
-CREATE MATERIALIZED VIEW user_stats AS
-SELECT count(*) AS total_users,
- count(
- CASE
- WHEN (last_login > (CURRENT_DATE - '30 days'::interval)) THEN 1
- ELSE NULL::integer
- END) AS active_users,
- avg(login_count) AS avg_login_count
-FROM public.users_mv
-WITH DATA;
-
--- Index on materialized view
-CREATE INDEX idx_user_stats_total ON user_stats(total_users);
-
--- Trigger function for audit logging
-CREATE OR REPLACE FUNCTION audit_trigger_function()
-RETURNS TRIGGER AS $$
-BEGIN
- INSERT INTO audit_log (table_name, operation, user_id, old_values, new_values)
- VALUES (
- TG_TABLE_NAME,
- TG_OP,
- COALESCE(NEW.id, OLD.id),
- CASE WHEN TG_OP = 'DELETE' THEN row_to_json(OLD) ELSE NULL END,
- CASE WHEN TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN row_to_json(NEW) ELSE NULL END
- );
- RETURN COALESCE(NEW, OLD);
-END;
-$$ LANGUAGE plpgsql;
-
--- Trigger on users table
-CREATE TRIGGER users_audit_trigger
- AFTER INSERT OR UPDATE OR DELETE ON users_mv
- FOR EACH ROW
- EXECUTE FUNCTION audit_trigger_function();
-
--- Stored procedure with parameters
-CREATE OR REPLACE PROCEDURE refresh_user_stats()
-LANGUAGE plpgsql AS $$
-BEGIN
- REFRESH MATERIALIZED VIEW user_stats;
- INSERT INTO audit_log (table_name, operation, timestamp)
- VALUES ('user_stats', 'REFRESH', CURRENT_TIMESTAMP);
-END;
-$$;
-`,
- },
- {
- name: "cross_schema_references",
- ddl: `
--- Create additional schemas
-CREATE SCHEMA hr;
-CREATE SCHEMA finance;
-
--- Tables in hr schema
-CREATE TABLE hr.departments (
- id SERIAL PRIMARY KEY,
- name VARCHAR(50) NOT NULL UNIQUE,
- manager_id INTEGER
-);
-
-CREATE TABLE hr.employees (
- id SERIAL PRIMARY KEY,
- first_name VARCHAR(50) NOT NULL,
- last_name VARCHAR(50) NOT NULL,
- department_id INTEGER NOT NULL,
- salary DECIMAL(10, 2),
- hire_date DATE DEFAULT CURRENT_DATE,
- CONSTRAINT fk_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id)
-);
-
--- Self-referencing foreign key for manager
-ALTER TABLE hr.departments
-ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES hr.employees(id);
-
--- Tables in finance schema
-CREATE TABLE finance.budgets (
- id SERIAL PRIMARY KEY,
- department_id INTEGER NOT NULL,
- fiscal_year INTEGER NOT NULL,
- allocated_amount DECIMAL(12, 2) NOT NULL,
- spent_amount DECIMAL(12, 2) DEFAULT 0.00,
- -- Cross-schema foreign key
- CONSTRAINT fk_budget_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id),
- CONSTRAINT unique_budget_year UNIQUE (department_id, fiscal_year)
-);
-
--- View that joins across schemas
-CREATE VIEW finance.department_spending AS
-SELECT d.name AS department_name,
- b.fiscal_year,
- b.allocated_amount,
- b.spent_amount,
- (b.allocated_amount - b.spent_amount) AS remaining_budget
-FROM (hr.departments d
- JOIN finance.budgets b ON ((d.id = b.department_id)));
-`,
- },
- {
- name: "advanced_indexes_and_constraints",
- ddl: `
-CREATE EXTENSION IF NOT EXISTS btree_gin;
-
-CREATE TABLE products (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- description TEXT,
- price DECIMAL(10, 2),
- category_tags TEXT[],
- metadata JSONB,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- sku VARCHAR(50),
- coordinates POINT
-);
-
--- BRIN index for time-series data
-CREATE INDEX idx_products_created_brin ON products USING BRIN (created_at);
-
--- Hash index
-CREATE INDEX idx_products_sku_hash ON products USING HASH (sku);
-
--- Covering index (INCLUDE clause)
-CREATE INDEX idx_products_category_include ON products (price) INCLUDE (name, description);
-
--- Partial index with complex condition
-CREATE INDEX idx_expensive_products ON products (price, category_tags)
-WHERE price > 100.00 AND array_length(category_tags, 1) > 0;
-
--- Expression index with functions
-CREATE INDEX idx_products_name_lower ON products (lower(name::text));
-CREATE INDEX idx_products_price_rounded ON products (round(price));
-
--- Multi-column GIN index
-CREATE INDEX idx_products_tags_meta ON products USING GIN (category_tags, metadata);
-
--- Simple check constraints (avoiding complex exclusion constraints)
-CREATE TABLE orders_advanced (
- id SERIAL PRIMARY KEY,
- product_id INTEGER NOT NULL,
- quantity INTEGER NOT NULL,
- order_date DATE DEFAULT CURRENT_DATE,
- CONSTRAINT fk_product FOREIGN KEY (product_id) REFERENCES products(id),
- CONSTRAINT valid_quantity CHECK (quantity > 0 AND quantity <= 1000),
- CONSTRAINT recent_order CHECK (order_date >= CURRENT_DATE - INTERVAL '1 year')
-);
-`,
- },
- {
- name: "geometric_and_network_types",
- ddl: `
--- Using built-in geometric and network types (no PostGIS required)
-CREATE TABLE locations (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- coordinates POINT,
- boundary_box BOX,
- service_area POLYGON,
- delivery_path PATH,
- center_point CIRCLE,
- route_line LSEG
-);
-
-CREATE TABLE network_devices (
- id SERIAL PRIMARY KEY,
- hostname VARCHAR(100) NOT NULL,
- ip_address INET,
- subnet CIDR,
- mac_address MACADDR,
- ipv6_address INET,
- device_config JSONB
-);
-
--- Geometric indexes (using GIST for geometric types)
-CREATE INDEX idx_locations_coordinates ON locations USING GIST (coordinates);
-CREATE INDEX idx_locations_service_area ON locations USING GIST (service_area);
-
--- Network type indexes
-CREATE INDEX idx_devices_ip ON network_devices (ip_address);
--- Skip GIST index on CIDR as it doesn't have a default operator class
-CREATE INDEX idx_devices_subnet ON network_devices (subnet);
-
--- Range types
-CREATE TABLE reservations (
- id SERIAL PRIMARY KEY,
- resource_id INTEGER NOT NULL,
- date_range DATERANGE NOT NULL,
- time_range TSRANGE,
- price_range NUMRANGE,
- capacity_range INT4RANGE
-);
-
--- Index on range types
-CREATE INDEX idx_reservations_date_range ON reservations USING GIST (date_range);
-
--- Full-text search types
-CREATE TABLE documents_fts (
- id SERIAL PRIMARY KEY,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- search_vector TSVECTOR,
- keywords TSQUERY
-);
-
--- Full-text search index
-CREATE INDEX idx_documents_search ON documents_fts USING GIN (search_vector);
-
--- Update trigger for full-text search
-CREATE OR REPLACE FUNCTION update_search_vector() RETURNS TRIGGER AS $$
-BEGIN
- NEW.search_vector := to_tsvector('english', COALESCE(NEW.title, '') || ' ' || COALESCE(NEW.content, ''));
- RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER trigger_update_search
- BEFORE INSERT OR UPDATE ON documents_fts
- FOR EACH ROW
- EXECUTE FUNCTION update_search_vector();
-`,
- },
- {
- name: "routing_rules_pattern",
- ddl: `
--- Test the routing rules pattern (view with INSERT/UPDATE/DELETE rules)
--- This is the exact pattern from the user's scenario
-
--- Create a regular table (not partitioned to avoid test issues)
-CREATE TABLE old_name (
- id BIGINT NOT NULL PRIMARY KEY,
- created_at TIMESTAMP WITH TIME ZONE NOT NULL,
- updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
- name VARCHAR(100),
- data JSONB,
- status VARCHAR(20) DEFAULT 'active'::character varying
-);
-
--- Create indexes
-CREATE INDEX idx_old_name_status ON old_name (status);
-CREATE INDEX idx_old_name_created ON old_name (created_at);
-
--- Create the view that will act as the new interface
-CREATE VIEW new_name AS
-SELECT
- id,
- created_at,
- updated_at,
- name,
- data,
- status
-FROM old_name;
-
--- Create routing rules to redirect operations from view to table
-CREATE RULE new_name_delete AS
- ON DELETE TO new_name
- DO INSTEAD DELETE FROM old_name
- WHERE old_name.id = OLD.id;
-
-CREATE RULE new_name_insert AS
- ON INSERT TO new_name
- DO INSTEAD INSERT INTO old_name (id, created_at, updated_at, name, data, status)
- VALUES (NEW.id, NEW.created_at, NEW.updated_at, NEW.name, NEW.data, NEW.status);
-
-CREATE RULE new_name_update AS
- ON UPDATE TO new_name
- DO INSTEAD UPDATE old_name
- SET
- created_at = NEW.created_at,
- updated_at = NEW.updated_at,
- name = NEW.name,
- data = NEW.data,
- status = NEW.status
- WHERE old_name.id = OLD.id;
-
-
--- Create another table with rules (not on a view)
-CREATE TABLE audit_table (
- id SERIAL PRIMARY KEY,
- table_name VARCHAR(50),
- operation VARCHAR(10),
- user_name VARCHAR(50),
- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE TABLE monitored_table (
- id SERIAL PRIMARY KEY,
- data TEXT
-);
-
--- Create a rule on a regular table (not a view)
-CREATE RULE log_monitored_inserts AS
- ON INSERT TO monitored_table
- DO ALSO INSERT INTO audit_table (table_name, operation, user_name)
- VALUES ('monitored_table', 'INSERT', current_user);
-
-CREATE RULE log_monitored_deletes AS
- ON DELETE TO monitored_table
- DO ALSO INSERT INTO audit_table (table_name, operation, user_name)
- VALUES ('monitored_table', 'DELETE', current_user);
-`,
- },
- {
- name: "table_inheritance_and_partitioning",
- ddl: `
--- Skip table inheritance as it's not supported
--- Just test partitioning features
-
--- List partitioning (without unique constraints that would cause issues)
-CREATE TABLE events (
- id BIGINT,
- event_type VARCHAR(20) NOT NULL,
- event_data JSONB,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-) PARTITION BY LIST (event_type);
-
-CREATE TABLE events_user PARTITION OF events
-FOR VALUES IN ('user_login', 'user_logout', 'user_register');
-
-CREATE TABLE events_system PARTITION OF events
-FOR VALUES IN ('system_start', 'system_stop', 'system_error');
-
-CREATE TABLE events_audit PARTITION OF events
-FOR VALUES IN ('data_change', 'permission_change', 'config_change');
-
--- Hash partitioning (without primary key to avoid partitioning column requirement)
-CREATE TABLE user_sessions (
- session_id UUID DEFAULT gen_random_uuid(),
- user_id INTEGER NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- last_activity TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- ip_address INET
-) PARTITION BY HASH (user_id);
-
-CREATE TABLE user_sessions_0 PARTITION OF user_sessions
-FOR VALUES WITH (modulus 4, remainder 0);
-
-CREATE TABLE user_sessions_1 PARTITION OF user_sessions
-FOR VALUES WITH (modulus 4, remainder 1);
-
-CREATE TABLE user_sessions_2 PARTITION OF user_sessions
-FOR VALUES WITH (modulus 4, remainder 2);
-
-CREATE TABLE user_sessions_3 PARTITION OF user_sessions
-FOR VALUES WITH (modulus 4, remainder 3);
-
--- Unlogged table for temporary data
-CREATE UNLOGGED TABLE temp_calculations (
- id SERIAL PRIMARY KEY,
- calculation_data JSONB,
- result DECIMAL(15, 6),
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-`,
- },
- {
- name: "array_types_comprehensive",
- ddl: `
--- Create custom enum for testing custom type arrays
-CREATE TYPE status_enum AS ENUM ('active', 'inactive', 'pending');
-CREATE TYPE priority_level AS ENUM ('low', 'medium', 'high', 'urgent');
-
--- Table with comprehensive array type coverage
-CREATE TABLE array_types_test (
- id SERIAL PRIMARY KEY,
-
- -- Integer array types (different aliases)
- int_array int[],
- int4_array int4[],
- integer_array integer[],
- bigint_array bigint[],
- int8_array int8[],
- smallint_array smallint[],
- int2_array int2[],
-
- -- Multi-dimensional arrays (should normalize to same as single dimension)
- int_multi_array int[][],
- text_multi_array text[][][],
-
- -- Floating point array types
- real_array real[],
- float4_array float4[],
- double_precision_array double precision[],
- float8_array float8[],
-
- -- Character array types with and without precision
- char_array char[],
- char_with_length char(10)[],
- varchar_array varchar[],
- varchar_with_length varchar(255)[],
- character_array character[],
- character_with_length character(50)[],
- character_varying_array character varying[],
- character_varying_with_length character varying(100)[],
- text_array text[],
-
- -- Boolean arrays
- bool_array bool[],
- boolean_array boolean[],
-
- -- Numeric arrays with precision
- numeric_array numeric[],
- numeric_with_precision numeric(10,2)[],
- decimal_array decimal[],
- decimal_with_precision decimal(15,4)[],
- money_array money[],
-
- -- Date/time arrays
- date_array date[],
- time_array time[],
- time_with_tz time with time zone[],
- timetz_array timetz[],
- timestamp_array timestamp[],
- timestamp_with_tz timestamp with time zone[],
- timestamptz_array timestamptz[],
- interval_array interval[],
-
- -- Binary and UUID arrays
- bytea_array bytea[],
- uuid_array uuid[],
-
- -- JSON arrays
- json_array json[],
- jsonb_array jsonb[],
-
- -- Network address arrays
- inet_array inet[],
- cidr_array cidr[],
- macaddr_array macaddr[],
- macaddr8_array macaddr8[],
-
- -- Geometric type arrays
- point_array point[],
- line_array line[],
- lseg_array lseg[],
- box_array box[],
- path_array path[],
- polygon_array polygon[],
- circle_array circle[],
-
- -- Full-text search arrays
- tsvector_array tsvector[],
- tsquery_array tsquery[],
-
- -- Range type arrays
- int4range_array int4range[],
- int8range_array int8range[],
- numrange_array numrange[],
- tsrange_array tsrange[],
- tstzrange_array tstzrange[],
- daterange_array daterange[],
-
- -- Multi-range type arrays (PostgreSQL 14+)
- int4multirange_array int4multirange[],
- int8multirange_array int8multirange[],
- nummultirange_array nummultirange[],
- tsmultirange_array tsmultirange[],
- tstzmultirange_array tstzmultirange[],
- datemultirange_array datemultirange[],
-
- -- Bit string arrays
- bit_array bit[],
- bit_with_length bit(8)[],
- bit_varying_array bit varying[],
- varbit_array varbit[],
- varbit_with_length varbit(16)[],
-
- -- Object identifier arrays
- oid_array oid[],
- regclass_array regclass[],
- regproc_array regproc[],
- regprocedure_array regprocedure[],
- regoper_array regoper[],
- regoperator_array regoperator[],
- regtype_array regtype[],
- regconfig_array regconfig[],
- regdictionary_array regdictionary[],
- regnamespace_array regnamespace[],
- regrole_array regrole[],
- regcollation_array regcollation[],
-
- -- System type arrays
- tid_array tid[],
- xid_array xid[],
- xid8_array xid8[],
- cid_array cid[],
- pg_lsn_array pg_lsn[],
-
- -- Custom enum arrays (should use fallback logic)
- status_array status_enum[],
- priority_array priority_level[],
-
- -- XML array
- xml_array xml[],
-
- -- Other special arrays (note: record[] not allowed in table columns)
- jsonpath_array jsonpath[],
-
- -- Additional system types (compatible with PostgreSQL 16)
- txid_snapshot_array txid_snapshot[],
-
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
--- Add some sample data to verify the schema works
-INSERT INTO array_types_test (
- int_array,
- varchar_with_length,
- bool_array,
- status_array,
- json_array
-) VALUES (
- ARRAY[1, 2, 3],
- ARRAY['test1', 'test2'],
- ARRAY[true, false, true],
- ARRAY['active'::status_enum, 'pending'::status_enum],
- ARRAY['{"key": "value1"}'::json, '{"key": "value2"}'::json]
-);
-`,
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- t.Parallel()
-
- // Create a new database for each test case with unique name
- dbName := fmt.Sprintf("test_%s_%d", strings.ReplaceAll(tc.name, "-", "_"), time.Now().UnixNano()%1000000)
- _, err := container.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))
- require.NoError(t, err)
-
- // Create driver for the test database
- driver, err := createPgDriver(ctx, container.GetHost(), container.GetPort(), dbName)
- require.NoError(t, err)
- defer driver.Close(ctx)
-
- // Execute the DDL
- _, err = driver.GetDB().ExecContext(ctx, tc.ddl)
- require.NoError(t, err)
-
- // Get metadata using Driver.SyncDBSchema
- syncMetadata, err := driver.SyncDBSchema(ctx)
- require.NoError(t, err)
-
- // Get metadata using get_database_metadata
- parseMetadata, err := GetDatabaseMetadata(tc.ddl)
- require.NoError(t, err)
-
- // Compare the two metadata structures
- compareMetadata(t, syncMetadata, parseMetadata)
-
- // Additional validation: use schema differ to ensure no differences detected
- validateWithSchemaDiffer(t, tc.name, syncMetadata, parseMetadata)
-
- // Validate rules are handled correctly (no cycles, proper dumping)
- validateRoutingRules(t, syncMetadata)
-
- // Additional validation for array types test case
- if tc.name == "array_types_comprehensive" {
- validateArrayTypes(t, syncMetadata, parseMetadata)
- }
- })
- }
-}
-
-// compareMetadata compares metadata from sync.go and get_database_metadata
-func compareMetadata(t *testing.T, syncMeta, parseMeta *storepb.DatabaseSchemaMetadata) {
- // Compare schemas
- require.Equal(t, len(syncMeta.Schemas), len(parseMeta.Schemas), "number of schemas should match")
-
- // Find the public schema in both
- var syncPublic, parsePublic *storepb.SchemaMetadata
- for _, schema := range syncMeta.Schemas {
- if schema.Name == "public" {
- syncPublic = schema
- break
- }
- }
- for _, schema := range parseMeta.Schemas {
- if schema.Name == "public" {
- parsePublic = schema
- break
- }
- }
-
- require.NotNil(t, syncPublic, "sync metadata should have public schema")
- require.NotNil(t, parsePublic, "parse metadata should have public schema")
-
- // Compare tables
- compareTables(t, syncPublic.Tables, parsePublic.Tables)
-
- // Compare views
- compareViews(t, syncPublic.Views, parsePublic.Views)
-
- // Compare materialized views
- compareMaterializedViews(t, syncPublic.MaterializedViews, parsePublic.MaterializedViews)
-
- // Compare functions
- compareFunctions(t, syncPublic.Functions, parsePublic.Functions)
-
- // Compare procedures (part of functions in metadata)
- compareProcedures(t, syncPublic.Procedures, parsePublic.Procedures)
-
- // Compare sequences
- compareSequences(t, syncPublic.Sequences, parsePublic.Sequences)
-
- // Compare enums
- compareEnums(t, syncPublic.EnumTypes, parsePublic.EnumTypes)
-
- // Note: Triggers are stored at the table level, not schema level
- // They will be compared as part of table comparison
-
- // Compare extensions
- compareExtensions(t, syncMeta.Extensions, parseMeta.Extensions)
-
- // Compare all schemas for cross-schema tests
- compareAllSchemas(t, syncMeta.Schemas, parseMeta.Schemas)
-}
-
-// normalizeSQL normalizes SQL for comparison by:
-// - Converting to lowercase
-// - Removing extra whitespace
-// - Removing trailing semicolons
-// - Removing schema qualifiers for common schemas
-// - Normalizing parentheses
-func normalizeSQL(sql string) string {
- // Convert to lowercase
- sql = strings.ToLower(sql)
-
- // Replace multiple spaces/newlines with single space
- sql = strings.Join(strings.Fields(sql), " ")
-
- // Remove trailing semicolons
- sql = strings.TrimSuffix(sql, ";")
-
- // Remove schema qualifiers for public schema
- // This handles cases like "public.table_name" -> "table_name"
- sql = strings.ReplaceAll(sql, "public.", "")
-
- // Handle PostgreSQL's tendency to wrap WHERE conditions in parentheses
- // e.g., "WHERE (condition)" -> "WHERE condition"
- // We need to be careful to only remove the outermost parentheses around the entire WHERE clause
- whereIndex := strings.Index(sql, "where (")
- if whereIndex >= 0 {
- // Find the matching closing parenthesis for the WHERE clause
- afterWhere := sql[whereIndex+7:] // Skip "where ("
- openCount := 1
- closeIndex := -1
-
- // Find the matching closing parenthesis
- for i, ch := range afterWhere {
- if ch == '(' {
- openCount++
- } else if ch == ')' {
- openCount--
- if openCount == 0 {
- closeIndex = i
- break
- }
- }
- }
-
- // If we found the matching closing parenthesis and it's at the end or followed by end/order/group
- if closeIndex >= 0 {
- beforeWhere := sql[:whereIndex+6] // Include "where "
- afterCloseParen := ""
- if whereIndex+7+closeIndex+1 < len(sql) {
- afterCloseParen = sql[whereIndex+7+closeIndex+1:]
- }
-
- // Check if the closing paren is at the end or followed by valid SQL keywords
- if afterCloseParen == "" || strings.HasPrefix(strings.TrimSpace(afterCloseParen), "order by") ||
- strings.HasPrefix(strings.TrimSpace(afterCloseParen), "group by") ||
- strings.HasPrefix(strings.TrimSpace(afterCloseParen), "limit") {
- // Remove the parentheses
- sql = beforeWhere + afterWhere[:closeIndex] + afterCloseParen
- }
- }
- }
-
- // PostgreSQL function and procedure-specific normalizations
- if strings.Contains(sql, "function") || strings.Contains(sql, "procedure") {
- // Normalize CREATE OR REPLACE vs CREATE
- sql = strings.ReplaceAll(sql, "create or replace function", "create function")
-
- // Normalize parameter types to consistent forms
- sql = strings.ReplaceAll(sql, "character varying", "varchar")
- sql = strings.ReplaceAll(sql, "returns numeric", "returns decimal")
-
- // Normalize dollar quoting
- sql = strings.ReplaceAll(sql, "$function$", "$$")
- sql = strings.ReplaceAll(sql, "$procedure$", "$$")
-
- // Move LANGUAGE clause to end for consistent comparison
- if strings.Contains(sql, "language plpgsql") && !strings.HasSuffix(sql, "language plpgsql") {
- withoutLanguage := strings.ReplaceAll(sql, " language plpgsql", "")
- withoutLanguage = strings.ReplaceAll(withoutLanguage, "language plpgsql ", "")
- sql = strings.TrimSpace(withoutLanguage) + " language plpgsql"
- }
- }
-
- // Final trim
- sql = strings.TrimSpace(sql)
-
- return sql
-}
-
-// normalizeSignature normalizes function signatures for comparison
-func normalizeSignature(sig string) string {
- // Convert to lowercase
- sig = strings.ToLower(sig)
-
- // Remove extra spaces
- sig = strings.Join(strings.Fields(sig), " ")
-
- // Remove spaces around parentheses and commas
- sig = strings.ReplaceAll(sig, " (", "(")
- sig = strings.ReplaceAll(sig, "( ", "(")
- sig = strings.ReplaceAll(sig, " )", ")")
- sig = strings.ReplaceAll(sig, ") ", ")")
- sig = strings.ReplaceAll(sig, " ,", ",")
- sig = strings.ReplaceAll(sig, ", ", ",")
-
- // Remove quotes around function names
- sig = strings.ReplaceAll(sig, "\"", "")
-
- return sig
-}
-
-func compareTables(t *testing.T, syncTables, parseTables []*storepb.TableMetadata) {
- // Log the tables found for debugging
- t.Logf("Sync tables: %d", len(syncTables))
- for _, table := range syncTables {
- t.Logf(" - %s", table.Name)
- }
- t.Logf("Parse tables: %d", len(parseTables))
- for _, table := range parseTables {
- t.Logf(" - %s", table.Name)
- }
-
- require.Equal(t, len(syncTables), len(parseTables), "number of tables should match")
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.TableMetadata)
- for _, table := range syncTables {
- syncMap[table.Name] = table
- }
-
- parseMap := make(map[string]*storepb.TableMetadata)
- for _, table := range parseTables {
- parseMap[table.Name] = table
- }
-
- // Compare each table
- for name, syncTable := range syncMap {
- parseTable, exists := parseMap[name]
- require.True(t, exists, "table %s should exist in parsed metadata", name)
-
- // Compare columns
- compareColumns(t, name, syncTable.Columns, parseTable.Columns)
-
- // Compare rules
- compareRules(t, name, syncTable.Rules, parseTable.Rules)
-
- // Compare indexes
- compareIndexes(t, name, syncTable.Indexes, parseTable.Indexes)
-
- // Compare foreign keys
- compareForeignKeys(t, name, syncTable.ForeignKeys, parseTable.ForeignKeys)
-
- // Compare partitions
- comparePartitions(t, name, syncTable.Partitions, parseTable.Partitions)
-
- // Compare triggers
- compareTriggers(t, name, syncTable.Triggers, parseTable.Triggers)
- }
-}
-
-func compareColumns(t *testing.T, tableName string, syncCols, parseCols []*storepb.ColumnMetadata) {
- require.Equal(t, len(syncCols), len(parseCols), "table %s: number of columns should match", tableName)
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.ColumnMetadata)
- for _, col := range syncCols {
- syncMap[col.Name] = col
- }
-
- for _, parseCol := range parseCols {
- syncCol, exists := syncMap[parseCol.Name]
- require.True(t, exists, "table %s: column %s should exist in sync metadata", tableName, parseCol.Name)
-
- // Compare column properties
- require.Equal(t, syncCol.Type, parseCol.Type, "table %s, column %s: type should match", tableName, parseCol.Name)
- require.Equal(t, syncCol.Nullable, parseCol.Nullable, "table %s, column %s: nullable should match", tableName, parseCol.Name)
-
- // Compare default values if both exist
- hasDefaultSync := syncCol.Default != ""
- hasDefaultParse := parseCol.Default != ""
- if hasDefaultSync && hasDefaultParse {
- // Default values might be represented differently, so we just check they exist
- t.Logf("table %s, column %s: default values exist in both", tableName, parseCol.Name)
- }
- }
-}
-
-func compareIndexes(t *testing.T, tableName string, syncIndexes, parseIndexes []*storepb.IndexMetadata) {
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.IndexMetadata)
- for _, idx := range syncIndexes {
- syncMap[idx.Name] = idx
- }
-
- parseMap := make(map[string]*storepb.IndexMetadata)
- for _, idx := range parseIndexes {
- parseMap[idx.Name] = idx
- }
-
- // Check that both directions have the same indexes
- require.Equal(t, len(syncIndexes), len(parseIndexes), "mismatch in number of indexes for table %s", tableName)
-
- // Compare indexes that exist in parse
- for name, parseIdx := range parseMap {
- syncIdx, exists := syncMap[name]
- if !exists {
- // Some indexes might be system-generated and not in DDL
- t.Logf("table %s: index %s exists in parse but not in sync (might be implicit)", tableName, name)
- continue
- }
-
- // 1. Name - explicitly validate name consistency
- require.Equal(t, syncIdx.Name, parseIdx.Name, "table %s, index %s: name should match", tableName, name)
-
- // 2. Primary - validate primary key flag
- require.Equal(t, syncIdx.Primary, parseIdx.Primary, "table %s, index %s: primary should match", tableName, name)
-
- // 3. Unique - validate unique constraint flag
- require.Equal(t, syncIdx.Unique, parseIdx.Unique, "table %s, index %s: unique should match", tableName, name)
-
- // 4. Type - validate index type
- if syncIdx.Type != "" || parseIdx.Type != "" {
- require.Equal(t, syncIdx.Type, parseIdx.Type, "table %s, index %s: type should match", tableName, name)
- }
-
- // 5. Expressions - compare expressions using AST-based semantic comparison
- if len(syncIdx.Expressions) == len(parseIdx.Expressions) {
- for i := range syncIdx.Expressions {
- // Use AST-based semantic comparison instead of string normalization
- equal := ast.CompareExpressionsSemantically(syncIdx.Expressions[i], parseIdx.Expressions[i])
- require.True(t, equal, "table %s, index %s: expression[%d] should be semantically equivalent. sync: %q, parse: %q",
- tableName, name, i, syncIdx.Expressions[i], parseIdx.Expressions[i])
- }
- } else {
- require.Equal(t, len(syncIdx.Expressions), len(parseIdx.Expressions), "table %s, index %s: expressions count should match", tableName, name)
- }
-
- // 6. Descending - compare descending order for each expression
- // Note: sync.go currently doesn't populate the Descending field, so we need to handle both cases
- if len(syncIdx.Descending) > 0 && len(parseIdx.Descending) > 0 {
- // Both have descending info, compare them
- require.Equal(t, len(syncIdx.Descending), len(parseIdx.Descending), "table %s, index %s: descending array length should match", tableName, name)
- for i := range syncIdx.Descending {
- require.Equal(t, syncIdx.Descending[i], parseIdx.Descending[i], "table %s, index %s: descending[%d] should match", tableName, name, i)
- }
- } else if len(parseIdx.Descending) > 0 {
- // Only parser has descending info, verify it matches the number of expressions
- require.Equal(t, len(parseIdx.Expressions), len(parseIdx.Descending), "table %s, index %s: descending array should match expressions count", tableName, name)
- }
-
- // 7. KeyLength - validate index key lengths (PostgreSQL specific - prefix lengths on expressions)
- if len(syncIdx.KeyLength) > 0 || len(parseIdx.KeyLength) > 0 {
- require.Equal(t, len(syncIdx.KeyLength), len(parseIdx.KeyLength), "table %s, index %s: key length array length should match", tableName, name)
- for i := range syncIdx.KeyLength {
- if i < len(parseIdx.KeyLength) {
- require.Equal(t, syncIdx.KeyLength[i], parseIdx.KeyLength[i], "table %s, index %s: key length[%d] should match", tableName, name, i)
- }
- }
- }
-
- // 8. Visible - validate index visibility (not commonly used in PostgreSQL, but validate if present)
- require.Equal(t, syncIdx.Visible, parseIdx.Visible, "table %s, index %s: visible should match", tableName, name)
-
- // 9. Comment - validate index comment
- if syncIdx.Comment != "" || parseIdx.Comment != "" {
- require.Equal(t, syncIdx.Comment, parseIdx.Comment, "table %s, index %s: comment should match", tableName, name)
- }
-
- // 10. IsConstraint - validate if index represents a constraint
- require.Equal(t, syncIdx.IsConstraint, parseIdx.IsConstraint, "table %s, index %s: IsConstraint should match", tableName, name)
-
- // 11. Definition - validate full index definition using AST-based semantic comparison
- if syncIdx.Definition != "" || parseIdx.Definition != "" {
- if syncIdx.Definition != "" && parseIdx.Definition != "" {
- definitionsEqual := compareIndexDefinitionsSemanticaly(syncIdx.Definition, parseIdx.Definition)
- require.True(t, definitionsEqual, "table %s, index %s: definition should match\nSync: %s\nParse: %s", tableName, name, syncIdx.Definition, parseIdx.Definition)
- }
- }
-
- t.Logf("✓ Validated all IndexMetadata fields for index %s: name=%s, primary=%v, unique=%v, type=%s, expressions=%v, visible=%v, comment=%s",
- name, parseIdx.Name, parseIdx.Primary, parseIdx.Unique, parseIdx.Type, parseIdx.Expressions, parseIdx.Visible, parseIdx.Comment)
- }
-}
-
-func compareForeignKeys(t *testing.T, tableName string, syncFKs, parseFKs []*storepb.ForeignKeyMetadata) {
- require.Equal(t, len(syncFKs), len(parseFKs), "table %s: number of foreign keys should match", tableName)
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.ForeignKeyMetadata)
- for _, fk := range syncFKs {
- syncMap[fk.Name] = fk
- }
-
- for _, parseFk := range parseFKs {
- syncFk, exists := syncMap[parseFk.Name]
- require.True(t, exists, "table %s: foreign key %s should exist in sync metadata", tableName, parseFk.Name)
-
- require.ElementsMatch(t, syncFk.Columns, parseFk.Columns, "table %s, FK %s: columns should match", tableName, parseFk.Name)
- require.Equal(t, syncFk.ReferencedTable, parseFk.ReferencedTable, "table %s, FK %s: referenced table should match", tableName, parseFk.Name)
- require.ElementsMatch(t, syncFk.ReferencedColumns, parseFk.ReferencedColumns, "table %s, FK %s: referenced columns should match", tableName, parseFk.Name)
- }
-}
-
-func comparePartitions(t *testing.T, tableName string, syncParts, parseParts []*storepb.TablePartitionMetadata) {
- require.Equal(t, len(syncParts), len(parseParts), "table %s: number of partitions should match", tableName)
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.TablePartitionMetadata)
- for _, part := range syncParts {
- syncMap[part.Name] = part
- }
-
- for _, parsePart := range parseParts {
- syncPart, exists := syncMap[parsePart.Name]
- require.True(t, exists, "table %s: partition %s should exist in sync metadata", tableName, parsePart.Name)
- // Use AST-based semantic comparison for partition expressions
- equal := ast.CompareExpressionsSemantically(syncPart.Expression, parsePart.Expression)
- require.True(t, equal, "table %s, partition %s: expression should be semantically equivalent. sync: %q, parse: %q",
- tableName, parsePart.Name, syncPart.Expression, parsePart.Expression)
- require.Equal(t, syncPart.Value, parsePart.Value, "table %s, partition %s: value should match", tableName, parsePart.Name)
- }
-}
-
-func compareViews(t *testing.T, syncViews, parseViews []*storepb.ViewMetadata) {
- require.Equal(t, len(syncViews), len(parseViews), "number of views should match")
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.ViewMetadata)
- for _, view := range syncViews {
- syncMap[view.Name] = view
- }
-
- for _, parseView := range parseViews {
- syncView, exists := syncMap[parseView.Name]
- require.True(t, exists, "view %s should exist in sync metadata", parseView.Name)
-
- // Compare view definitions using PostgreSQL view comparer for better normalization
- comparer := &PostgreSQLViewComparer{}
- definitionsEqual := comparer.compareViewsSemanticaly(syncView.Definition, parseView.Definition)
- require.True(t, definitionsEqual, "view %s: definition should match\nSync: %s\nParse: %s", parseView.Name, syncView.Definition, parseView.Definition)
-
- // Compare comment if present
- if syncView.Comment != "" || parseView.Comment != "" {
- require.Equal(t, syncView.Comment, parseView.Comment, "view %s: comment should match", parseView.Name)
- }
-
- // Compare rules
- compareRules(t, parseView.Name, syncView.Rules, parseView.Rules)
- }
-}
-
-func compareFunctions(t *testing.T, syncFuncs, parseFuncs []*storepb.FunctionMetadata) {
- // Function comparison is tricky because signatures might be formatted differently
- t.Logf("sync has %d functions, parse has %d functions", len(syncFuncs), len(parseFuncs))
-
- // Currently the parser doesn't extract functions from DDL, so we expect 0 functions from parser
- // If parser starts extracting functions, we should implement proper comparison here
- if len(parseFuncs) == 0 && len(syncFuncs) > 0 {
- // This is expected - parser doesn't extract functions yet
- return
- }
-
- // If parser starts returning functions, implement full comparison
- require.Equal(t, len(syncFuncs), len(parseFuncs), "number of functions should match")
-
- // Create maps for easier comparison - use function signature for mapping
- syncMap := make(map[string]*storepb.FunctionMetadata)
- for _, fn := range syncFuncs {
- syncMap[fn.Signature] = fn
- }
-
- for _, parseFn := range parseFuncs {
- // Try to find matching function by signature
- var syncFn *storepb.FunctionMetadata
- for _, sf := range syncFuncs {
- if normalizeSignature(sf.Signature) == normalizeSignature(parseFn.Signature) {
- syncFn = sf
- break
- }
- }
-
- require.NotNil(t, syncFn, "function with signature %s should exist in sync metadata", parseFn.Signature)
-
- // Compare function definitions using PostgreSQL function comparer
- comparer := &PostgreSQLFunctionComparer{}
- syncFunc := &storepb.FunctionMetadata{Definition: syncFn.Definition}
- parseFunc := &storepb.FunctionMetadata{Definition: parseFn.Definition}
-
- // Use function comparer to check if they are equal
- if !comparer.Equal(syncFunc, parseFunc) {
- // Get detailed comparison for better error message
- result, err := comparer.CompareDetailed(syncFunc, parseFunc)
- if err != nil {
- t.Errorf("function %s: error comparing functions: %v", parseFn.Name, err)
- } else if result != nil {
- t.Errorf("function %s: definitions differ - SignatureChanged: %v, BodyChanged: %v, AttributesChanged: %v",
- parseFn.Name, result.SignatureChanged, result.BodyChanged, result.AttributesChanged)
- t.Logf(" Sync definition: %s", syncFn.Definition)
- t.Logf(" Parse definition: %s", parseFn.Definition)
- }
- }
-
- // Compare comment if present
- if syncFn.Comment != "" || parseFn.Comment != "" {
- require.Equal(t, syncFn.Comment, parseFn.Comment, "function %s: comment should match", parseFn.Name)
- }
- }
-}
-
-func compareSequences(t *testing.T, syncSeqs, parseSeqs []*storepb.SequenceMetadata) {
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.SequenceMetadata)
- for _, seq := range syncSeqs {
- syncMap[seq.Name] = seq
- }
-
- parseMap := make(map[string]*storepb.SequenceMetadata)
- for _, seq := range parseSeqs {
- parseMap[seq.Name] = seq
- }
-
- // Check sequences in parseSeqs
- for _, parseSeq := range parseSeqs {
- syncSeq, exists := syncMap[parseSeq.Name]
- if !exists {
- // SERIAL columns create implicit sequences that might not be in DDL
- t.Logf("sequence %s exists in parse but not in sync (might be implicit from SERIAL)", parseSeq.Name)
- continue
- }
-
- // Compare basic properties
- if parseSeq.Start != "" {
- require.Equal(t, syncSeq.Start, parseSeq.Start, "sequence %s: start value should match", parseSeq.Name)
- }
- if parseSeq.Increment != "" {
- require.Equal(t, syncSeq.Increment, parseSeq.Increment, "sequence %s: increment should match", parseSeq.Name)
- }
- }
-
- // Check sequences in syncSeqs that are not in parseSeqs
- for _, syncSeq := range syncSeqs {
- _, exists := parseMap[syncSeq.Name]
- if !exists {
- // Skip implicit sequences created by SERIAL columns
- if strings.Contains(syncSeq.Name, "_id_seq") || strings.Contains(syncSeq.Name, "_seq") {
- t.Logf("sequence %s exists in sync but not in parse (implicit sequence from SERIAL column)", syncSeq.Name)
- continue
- }
- // For explicitly created sequences, this is an error
- require.True(t, exists, "sequence %s should exist in parsed metadata", syncSeq.Name)
- }
- }
-}
-
-func compareEnums(t *testing.T, syncEnums, parseEnums []*storepb.EnumTypeMetadata) {
- require.Equal(t, len(syncEnums), len(parseEnums), "number of enums should match")
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.EnumTypeMetadata)
- for _, enum := range syncEnums {
- syncMap[enum.Name] = enum
- }
-
- for _, parseEnum := range parseEnums {
- syncEnum, exists := syncMap[parseEnum.Name]
- require.True(t, exists, "enum %s should exist in sync metadata", parseEnum.Name)
- require.ElementsMatch(t, syncEnum.Values, parseEnum.Values, "enum %s: values should match", parseEnum.Name)
- }
-}
-
-func compareExtensions(t *testing.T, syncExts, parseExts []*storepb.ExtensionMetadata) {
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.ExtensionMetadata)
- for _, ext := range syncExts {
- syncMap[ext.Name] = ext
- }
-
- for _, parseExt := range parseExts {
- syncExt, exists := syncMap[parseExt.Name]
- require.True(t, exists, "extension %s should exist in sync metadata", parseExt.Name)
- require.Equal(t, syncExt.Schema, parseExt.Schema, "extension %s: schema should match", parseExt.Name)
- }
-}
-
-func compareMaterializedViews(t *testing.T, syncMViews, parseMViews []*storepb.MaterializedViewMetadata) {
- // Materialized views are not currently supported by the parser
- // The parser may incorrectly classify them as tables, so we handle this gracefully
- if len(parseMViews) == 0 && len(syncMViews) > 0 {
- t.Logf("Parser doesn't extract materialized views yet - found %d in sync", len(syncMViews))
- return
- }
-
- require.Equal(t, len(syncMViews), len(parseMViews), "number of materialized views should match")
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.MaterializedViewMetadata)
- for _, mv := range syncMViews {
- syncMap[mv.Name] = mv
- }
-
- for _, parseMv := range parseMViews {
- syncMv, exists := syncMap[parseMv.Name]
- require.True(t, exists, "materialized view %s should exist in sync metadata", parseMv.Name)
-
- // Compare definitions using PostgreSQL view comparer for better normalization
- comparer := &PostgreSQLViewComparer{}
- definitionsEqual := comparer.compareViewsSemanticaly(syncMv.Definition, parseMv.Definition)
- require.True(t, definitionsEqual, "materialized view %s: definition should match\nSync: %s\nParse: %s", parseMv.Name, syncMv.Definition, parseMv.Definition)
-
- // Compare comment if present
- if syncMv.Comment != "" || parseMv.Comment != "" {
- require.Equal(t, syncMv.Comment, parseMv.Comment, "materialized view %s: comment should match", parseMv.Name)
- }
-
- // Compare indexes on materialized views if present
- if len(syncMv.Indexes) > 0 || len(parseMv.Indexes) > 0 {
- compareIndexes(t, parseMv.Name, syncMv.Indexes, parseMv.Indexes)
- }
-
- // Compare triggers on materialized views if present
- if len(syncMv.Triggers) > 0 || len(parseMv.Triggers) > 0 {
- compareTriggers(t, parseMv.Name, syncMv.Triggers, parseMv.Triggers)
- }
- }
-}
-
-func compareProcedures(t *testing.T, syncProcs, parseProcs []*storepb.ProcedureMetadata) {
- // Procedures might not be extracted by parser yet, so handle gracefully
- if len(parseProcs) == 0 && len(syncProcs) > 0 {
- t.Logf("Parser doesn't extract procedures yet - found %d in sync", len(syncProcs))
- return
- }
-
- require.Equal(t, len(syncProcs), len(parseProcs), "number of procedures should match")
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.ProcedureMetadata)
- for _, proc := range syncProcs {
- syncMap[proc.Name] = proc
- }
-
- for _, parseProc := range parseProcs {
- syncProc, exists := syncMap[parseProc.Name]
- require.True(t, exists, "procedure %s should exist in sync metadata", parseProc.Name)
-
- // Compare definitions
- syncDef := normalizeSQL(syncProc.Definition)
- parseDef := normalizeSQL(parseProc.Definition)
- require.Equal(t, syncDef, parseDef, "procedure %s: definition should match", parseProc.Name)
- }
-}
-
-func compareRules(t *testing.T, objectName string, syncRules, parseRules []*storepb.RuleMetadata) {
- // Rules might not be extracted by parser yet, so handle gracefully
- if len(parseRules) == 0 && len(syncRules) > 0 {
- t.Logf("Object %s: Parser doesn't extract rules yet - found %d in sync", objectName, len(syncRules))
- return
- }
-
- require.Equal(t, len(syncRules), len(parseRules), "object %s: number of rules should match", objectName)
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.RuleMetadata)
- for _, rule := range syncRules {
- syncMap[rule.Name] = rule
- }
-
- for _, parseRule := range parseRules {
- syncRule, exists := syncMap[parseRule.Name]
- require.True(t, exists, "object %s: rule %s should exist in sync metadata", objectName, parseRule.Name)
-
- // Compare rule properties
- require.Equal(t, syncRule.Event, parseRule.Event, "object %s: rule %s: event should match", objectName, parseRule.Name)
- require.Equal(t, syncRule.IsInstead, parseRule.IsInstead, "object %s: rule %s: is_instead should match", objectName, parseRule.Name)
- require.Equal(t, syncRule.IsEnabled, parseRule.IsEnabled, "object %s: rule %s: is_enabled should match", objectName, parseRule.Name)
-
- // Compare conditions (might be empty)
- if syncRule.Condition != "" || parseRule.Condition != "" {
- require.Equal(t, syncRule.Condition, parseRule.Condition, "object %s: rule %s: condition should match", objectName, parseRule.Name)
- }
-
- // Compare definitions - normalize them first
- syncDef := normalizeSQL(syncRule.Definition)
- parseDef := normalizeSQL(parseRule.Definition)
- require.Equal(t, syncDef, parseDef, "object %s: rule %s: definition should match", objectName, parseRule.Name)
- }
-}
-
-func compareTriggers(t *testing.T, tableName string, syncTriggers, parseTriggers []*storepb.TriggerMetadata) {
- // Triggers might not be extracted by parser yet, so handle gracefully
- if len(parseTriggers) == 0 && len(syncTriggers) > 0 {
- t.Logf("Table %s: Parser doesn't extract triggers yet - found %d in sync", tableName, len(syncTriggers))
- return
- }
-
- require.Equal(t, len(syncTriggers), len(parseTriggers), "table %s: number of triggers should match", tableName)
-
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.TriggerMetadata)
- for _, trigger := range syncTriggers {
- syncMap[trigger.Name] = trigger
- }
-
- for _, parseTrigger := range parseTriggers {
- syncTrigger, exists := syncMap[parseTrigger.Name]
- require.True(t, exists, "table %s: trigger %s should exist in sync metadata", tableName, parseTrigger.Name)
-
- // Compare basic trigger properties
- require.Equal(t, syncTrigger.Event, parseTrigger.Event, "table %s, trigger %s: event should match", tableName, parseTrigger.Name)
- require.Equal(t, syncTrigger.Timing, parseTrigger.Timing, "table %s, trigger %s: timing should match", tableName, parseTrigger.Name)
-
- // Compare trigger body/definition if available
- if syncTrigger.Body != "" || parseTrigger.Body != "" {
- syncBody := normalizeSQL(syncTrigger.Body)
- parseBody := normalizeSQL(parseTrigger.Body)
- require.Equal(t, syncBody, parseBody, "table %s, trigger %s: body should match", tableName, parseTrigger.Name)
- }
- }
-}
-
-func compareAllSchemas(t *testing.T, syncSchemas, parseSchemas []*storepb.SchemaMetadata) {
- // For cross-schema test cases, we need to compare schemas beyond just 'public'
- // Create maps for easier comparison
- syncMap := make(map[string]*storepb.SchemaMetadata)
- for _, schema := range syncSchemas {
- syncMap[schema.Name] = schema
- }
-
- parseMap := make(map[string]*storepb.SchemaMetadata)
- for _, schema := range parseSchemas {
- parseMap[schema.Name] = schema
- }
-
- // Check that important schemas exist in both
- for _, parseSchema := range parseSchemas {
- if parseSchema.Name == "information_schema" || parseSchema.Name == "pg_catalog" {
- // Skip system schemas
- continue
- }
-
- syncSchema, exists := syncMap[parseSchema.Name]
- if !exists {
- t.Logf("Schema %s exists in parse but not in sync (might be expected for some test cases)", parseSchema.Name)
- continue
- }
-
- // Compare schema-specific content
- t.Logf("Comparing schema: %s", parseSchema.Name)
-
- // Compare tables in this schema
- compareTables(t, syncSchema.Tables, parseSchema.Tables)
-
- // Compare views in this schema
- compareViews(t, syncSchema.Views, parseSchema.Views)
-
- // Compare functions in this schema
- compareFunctions(t, syncSchema.Functions, parseSchema.Functions)
- }
-}
-
-// validateWithSchemaDiffer validates that the schema differ returns no significant differences between sync and parse metadata
-func validateWithSchemaDiffer(t *testing.T, testName string, syncMeta, parseMeta *storepb.DatabaseSchemaMetadata) {
- // Convert metadata to model.DatabaseMetadata for differ
- syncSchema := model.NewDatabaseMetadata(syncMeta, nil, nil, storepb.Engine_POSTGRES, false)
- parseSchema := model.NewDatabaseMetadata(parseMeta, nil, nil, storepb.Engine_POSTGRES, false)
-
- // Get schema diff
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, syncSchema, parseSchema)
- require.NoError(t, err, "test case %s: schema differ should not error", testName)
-
- if diff == nil {
- // No diff is expected - this is good
- return
- }
-
- // Check that all diff categories are empty - DDL parser should fully match PostgreSQL behavior
- var diffMessages []string
-
- if len(diff.SchemaChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d schema changes", len(diff.SchemaChanges)))
- for _, change := range diff.SchemaChanges {
- t.Logf("Schema change: %s %s", change.Action, change.SchemaName)
- }
- }
-
- if len(diff.TableChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d table changes", len(diff.TableChanges)))
- for _, change := range diff.TableChanges {
- t.Logf("Table change: %s %s.%s", change.Action, change.SchemaName, change.TableName)
- // Log detailed changes within the table
- if len(change.ColumnChanges) > 0 {
- t.Logf(" Column changes: %d", len(change.ColumnChanges))
- for _, colChange := range change.ColumnChanges {
- if colChange.OldColumn != nil && colChange.NewColumn != nil {
- t.Logf(" %s column %s: %s -> %s", colChange.Action, colChange.NewColumn.Name, colChange.OldColumn.Type, colChange.NewColumn.Type)
- if colChange.OldColumn.Default != colChange.NewColumn.Default {
- t.Logf(" Default changed: '%s' -> '%s'", colChange.OldColumn.Default, colChange.NewColumn.Default)
- }
- if colChange.OldColumn.Nullable != colChange.NewColumn.Nullable {
- t.Logf(" Nullable changed: %t -> %t", colChange.OldColumn.Nullable, colChange.NewColumn.Nullable)
- }
- } else if colChange.NewColumn != nil {
- t.Logf(" %s column %s: %s", colChange.Action, colChange.NewColumn.Name, colChange.NewColumn.Type)
- } else if colChange.OldColumn != nil {
- t.Logf(" %s column %s: %s", colChange.Action, colChange.OldColumn.Name, colChange.OldColumn.Type)
- }
- }
- }
- if len(change.IndexChanges) > 0 {
- t.Logf(" Index changes: %d", len(change.IndexChanges))
- for i, idxChange := range change.IndexChanges {
- // Look for DROP/CREATE pairs that indicate a difference was detected
- if i+1 < len(change.IndexChanges) && idxChange.Action == "DROP" && change.IndexChanges[i+1].Action == "CREATE" &&
- idxChange.OldIndex.Name == change.IndexChanges[i+1].NewIndex.Name {
- // This is a DROP/CREATE pair - log detailed comparison
- oldIdx := idxChange.OldIndex
- newIdx := change.IndexChanges[i+1].NewIndex
- t.Logf(" DETAILED INDEX COMPARISON for %s (DROP/CREATE pair):", oldIdx.Name)
- t.Logf(" Type: old='%s' vs new='%s' (equal: %v)", oldIdx.Type, newIdx.Type, oldIdx.Type == newIdx.Type)
- t.Logf(" Unique: old=%v vs new=%v (equal: %v)", oldIdx.Unique, newIdx.Unique, oldIdx.Unique == newIdx.Unique)
- t.Logf(" Primary: old=%v vs new=%v (equal: %v)", oldIdx.Primary, newIdx.Primary, oldIdx.Primary == newIdx.Primary)
- t.Logf(" Expressions: old=%v vs new=%v (equal: %v)", oldIdx.Expressions, newIdx.Expressions, fmt.Sprintf("%v", oldIdx.Expressions) == fmt.Sprintf("%v", newIdx.Expressions))
- t.Logf(" KeyLength: old=%v vs new=%v (equal: %v)", oldIdx.KeyLength, newIdx.KeyLength, fmt.Sprintf("%v", oldIdx.KeyLength) == fmt.Sprintf("%v", newIdx.KeyLength))
- t.Logf(" Descending: old=%v vs new=%v (equal: %v)", oldIdx.Descending, newIdx.Descending, fmt.Sprintf("%v", oldIdx.Descending) == fmt.Sprintf("%v", newIdx.Descending))
- t.Logf(" Visible: old=%v vs new=%v (equal: %v)", oldIdx.Visible, newIdx.Visible, oldIdx.Visible == newIdx.Visible)
- t.Logf(" Definition: old='%s' vs new='%s' (equal: %v)", oldIdx.Definition, newIdx.Definition, oldIdx.Definition == newIdx.Definition)
- t.Logf(" IsConstraint: old=%v vs new=%v (equal: %v)", oldIdx.IsConstraint, newIdx.IsConstraint, oldIdx.IsConstraint == newIdx.IsConstraint)
- }
-
- if idxChange.NewIndex != nil && idxChange.OldIndex != nil {
- // Log detailed comparison for ALTER cases
- t.Logf(" DETAILED INDEX COMPARISON for %s:", idxChange.NewIndex.Name)
- t.Logf(" Type: old='%s' vs new='%s' (equal: %v)", idxChange.OldIndex.Type, idxChange.NewIndex.Type, idxChange.OldIndex.Type == idxChange.NewIndex.Type)
- t.Logf(" Unique: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Unique, idxChange.NewIndex.Unique, idxChange.OldIndex.Unique == idxChange.NewIndex.Unique)
- t.Logf(" Primary: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Primary, idxChange.NewIndex.Primary, idxChange.OldIndex.Primary == idxChange.NewIndex.Primary)
- t.Logf(" Expressions: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Expressions, idxChange.NewIndex.Expressions, fmt.Sprintf("%v", idxChange.OldIndex.Expressions) == fmt.Sprintf("%v", idxChange.NewIndex.Expressions))
- t.Logf(" KeyLength: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.KeyLength, idxChange.NewIndex.KeyLength, fmt.Sprintf("%v", idxChange.OldIndex.KeyLength) == fmt.Sprintf("%v", idxChange.NewIndex.KeyLength))
- t.Logf(" Descending: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Descending, idxChange.NewIndex.Descending, fmt.Sprintf("%v", idxChange.OldIndex.Descending) == fmt.Sprintf("%v", idxChange.NewIndex.Descending))
- t.Logf(" Visible: old=%v vs new=%v (equal: %v)", idxChange.OldIndex.Visible, idxChange.NewIndex.Visible, idxChange.OldIndex.Visible == idxChange.NewIndex.Visible)
- t.Logf(" Definition: old='%s' vs new='%s' (equal: %v)", idxChange.OldIndex.Definition, idxChange.NewIndex.Definition, idxChange.OldIndex.Definition == idxChange.NewIndex.Definition)
- }
-
- if idxChange.NewIndex != nil {
- t.Logf(" %s index %s: type=%s, unique=%v, primary=%v, expressions=%v, definition=%s",
- idxChange.Action, idxChange.NewIndex.Name, idxChange.NewIndex.Type,
- idxChange.NewIndex.Unique, idxChange.NewIndex.Primary, idxChange.NewIndex.Expressions, idxChange.NewIndex.Definition)
- } else if idxChange.OldIndex != nil {
- t.Logf(" %s index %s: type=%s, unique=%v, primary=%v, expressions=%v, definition=%s",
- idxChange.Action, idxChange.OldIndex.Name, idxChange.OldIndex.Type,
- idxChange.OldIndex.Unique, idxChange.OldIndex.Primary, idxChange.OldIndex.Expressions, idxChange.OldIndex.Definition)
- }
- }
- }
- if len(change.ForeignKeyChanges) > 0 {
- t.Logf(" Foreign key changes: %d", len(change.ForeignKeyChanges))
- for _, fkChange := range change.ForeignKeyChanges {
- if fkChange.NewForeignKey != nil {
- t.Logf(" %s FK %s: columns=%v -> %s.%s(%v), onDelete=%s",
- fkChange.Action, fkChange.NewForeignKey.Name, fkChange.NewForeignKey.Columns,
- fkChange.NewForeignKey.ReferencedSchema, fkChange.NewForeignKey.ReferencedTable,
- fkChange.NewForeignKey.ReferencedColumns, fkChange.NewForeignKey.OnDelete)
- } else if fkChange.OldForeignKey != nil {
- t.Logf(" %s FK %s: columns=%v -> %s.%s(%v), onDelete=%s",
- fkChange.Action, fkChange.OldForeignKey.Name, fkChange.OldForeignKey.Columns,
- fkChange.OldForeignKey.ReferencedSchema, fkChange.OldForeignKey.ReferencedTable,
- fkChange.OldForeignKey.ReferencedColumns, fkChange.OldForeignKey.OnDelete)
- }
- }
- }
- if len(change.CheckConstraintChanges) > 0 {
- t.Logf(" Check constraint changes: %d", len(change.CheckConstraintChanges))
- for _, checkChange := range change.CheckConstraintChanges {
- if checkChange.NewCheckConstraint != nil && checkChange.OldCheckConstraint != nil {
- t.Logf(" %s CHECK %s: old='%s' -> new='%s'",
- checkChange.Action, checkChange.NewCheckConstraint.Name,
- checkChange.OldCheckConstraint.Expression, checkChange.NewCheckConstraint.Expression)
- } else if checkChange.NewCheckConstraint != nil {
- t.Logf(" %s CHECK %s: expression='%s'",
- checkChange.Action, checkChange.NewCheckConstraint.Name, checkChange.NewCheckConstraint.Expression)
- } else if checkChange.OldCheckConstraint != nil {
- t.Logf(" %s CHECK %s: expression='%s'",
- checkChange.Action, checkChange.OldCheckConstraint.Name, checkChange.OldCheckConstraint.Expression)
- }
- }
- }
- if len(change.TriggerChanges) > 0 {
- t.Logf(" Trigger changes: %d", len(change.TriggerChanges))
- for _, triggerChange := range change.TriggerChanges {
- if triggerChange.NewTrigger != nil && triggerChange.OldTrigger != nil {
- t.Logf(" %s TRIGGER %s: old='%s' -> new='%s'",
- triggerChange.Action, triggerChange.NewTrigger.Name,
- triggerChange.OldTrigger.Body, triggerChange.NewTrigger.Body)
- } else if triggerChange.NewTrigger != nil {
- t.Logf(" %s TRIGGER %s: body='%s'",
- triggerChange.Action, triggerChange.NewTrigger.Name, triggerChange.NewTrigger.Body)
- } else if triggerChange.OldTrigger != nil {
- t.Logf(" %s TRIGGER %s: body='%s'",
- triggerChange.Action, triggerChange.OldTrigger.Name, triggerChange.OldTrigger.Body)
- }
- }
- }
- }
- }
-
- if len(diff.ViewChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d view changes", len(diff.ViewChanges)))
- for _, change := range diff.ViewChanges {
- t.Logf("View change: %s %s.%s", change.Action, change.SchemaName, change.ViewName)
- }
- }
-
- if len(diff.MaterializedViewChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d materialized view changes", len(diff.MaterializedViewChanges)))
- for _, change := range diff.MaterializedViewChanges {
- t.Logf("Materialized view change: %s %s.%s", change.Action, change.SchemaName, change.MaterializedViewName)
- }
- }
-
- if len(diff.FunctionChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d function changes", len(diff.FunctionChanges)))
- for _, change := range diff.FunctionChanges {
- t.Logf("Function change: %s %s.%s", change.Action, change.SchemaName, change.FunctionName)
- // Log detailed function comparison for debugging
- if change.OldFunction != nil && change.NewFunction != nil {
- t.Logf(" Old function signature: %s", change.OldFunction.Signature)
- t.Logf(" New function signature: %s", change.NewFunction.Signature)
- t.Logf(" Old function definition: %s", change.OldFunction.Definition)
- t.Logf(" New function definition: %s", change.NewFunction.Definition)
- t.Logf(" Definitions equal: %v", change.OldFunction.Definition == change.NewFunction.Definition)
- t.Logf(" Signatures equal: %v", change.OldFunction.Signature == change.NewFunction.Signature)
- }
- }
- }
-
- if len(diff.ProcedureChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d procedure changes", len(diff.ProcedureChanges)))
- for _, change := range diff.ProcedureChanges {
- t.Logf("Procedure change: %s %s.%s", change.Action, change.SchemaName, change.ProcedureName)
- }
- }
-
- if len(diff.SequenceChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d sequence changes", len(diff.SequenceChanges)))
- for _, change := range diff.SequenceChanges {
- t.Logf("Sequence change: %s %s.%s", change.Action, change.SchemaName, change.SequenceName)
- if change.OldSequence != nil && change.NewSequence == nil {
- t.Logf(" ISSUE: SERIAL sequence %s missing in parsed metadata", change.SequenceName)
- } else if change.NewSequence != nil && change.OldSequence == nil {
- t.Logf(" ISSUE: Unexpected sequence %s in parsed metadata", change.SequenceName)
- }
- }
- }
-
- if len(diff.EnumTypeChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d enum type changes", len(diff.EnumTypeChanges)))
- for _, change := range diff.EnumTypeChanges {
- t.Logf("Enum type change: %s %s.%s", change.Action, change.SchemaName, change.EnumTypeName)
- }
- }
-
- if len(diff.EventChanges) > 0 {
- diffMessages = append(diffMessages, fmt.Sprintf("%d event changes", len(diff.EventChanges)))
- for _, change := range diff.EventChanges {
- t.Logf("Event change: %s %s", change.Action, change.EventName)
- }
- }
-
- // If we have any differences, fail the test - DDL parser must match PostgreSQL exactly
- if len(diffMessages) > 0 {
- require.Fail(t, fmt.Sprintf("test case %s: DDL parser should fully replicate PostgreSQL behavior. Differences found: %s", testName, strings.Join(diffMessages, ", ")))
- }
-}
-
-// validateRoutingRules validates that rules are handled correctly for all test cases
-func validateRoutingRules(t *testing.T, metadata *storepb.DatabaseSchemaMetadata) {
- // This validation ensures:
- // 1. No cycle errors occur with rules (if we got here, it passed)
- // 2. GetDatabaseDefinition works without errors
- // 3. Rules are properly included in the output
-
- require.NotNil(t, metadata, "Should have metadata without cycle errors")
-
- // Test that GetDatabaseDefinition works without cycle errors
- ctx := schema.GetDefinitionContext{
- PrintHeader: false,
- }
- definition, err := GetDatabaseDefinition(ctx, metadata)
- require.NoError(t, err, "GetDatabaseDefinition should not error even with rules")
- require.NotEmpty(t, definition)
-
- // Count rules across all schemas, tables, and views
- var totalRules int
- var hasRules bool
-
- for _, schema := range metadata.Schemas {
- // Check tables for rules
- for _, table := range schema.Tables {
- if len(table.Rules) > 0 {
- hasRules = true
- totalRules += len(table.Rules)
-
- // Verify each rule is in the output
- for _, rule := range table.Rules {
- require.NotEmpty(t, rule.Definition, "Rule should have definition")
- require.NotEmpty(t, rule.Event, "Rule should have event type")
- // Verify the rule appears in the output
- if !strings.Contains(definition, rule.Name) {
- t.Logf("Warning: Rule %s not found in output (might be a system rule)", rule.Name)
- }
- }
- }
- }
-
- // Check views for rules
- for _, view := range schema.Views {
- if len(view.Rules) > 0 {
- hasRules = true
- totalRules += len(view.Rules)
-
- // Verify each rule is in the output
- for _, rule := range view.Rules {
- require.NotEmpty(t, rule.Definition, "Rule should have definition")
- require.NotEmpty(t, rule.Event, "Rule should have event type")
- // Non-SELECT rules should be in the output
- if rule.Event != "SELECT" && !strings.Contains(definition, rule.Name) {
- t.Logf("Warning: Rule %s not found in output", rule.Name)
- }
- }
- }
- }
- }
-
- // Log summary for debugging
- if hasRules {
- t.Logf("✓ Found %d rules in metadata, GetDatabaseDefinition succeeded without cycles", totalRules)
-
- // For test cases with specific routing patterns, do additional validation
- // Check for the routing rules pattern (view with INSERT/UPDATE/DELETE rules)
- for _, schema := range metadata.Schemas {
- for _, view := range schema.Views {
- if view.Name == "new_name" && len(view.Rules) > 0 {
- // This is the routing pattern - verify correct ordering
- tablePos := strings.Index(definition, `CREATE TABLE "public"."old_name"`)
- viewPos := strings.Index(definition, `CREATE VIEW "public"."new_name"`)
- if tablePos > -1 && viewPos > -1 {
- require.Less(t, tablePos, viewPos, "With routing rules: table should come before view due to dependency")
- t.Log("✓ Routing rules pattern validated - correct ordering maintained")
- }
- }
- }
- }
- } else {
- t.Log("No rules in this test case - validation passed")
- }
-}
-
-// compareIndexDefinitionsSemanticaly compares two PostgreSQL index definitions using AST-based semantic comparison
-func compareIndexDefinitionsSemanticaly(def1, def2 string) bool {
- if strings.TrimSpace(def1) == strings.TrimSpace(def2) {
- return true
- }
-
- // Parse the index definitions to extract components
- parts1 := parseIndexDefinition(def1)
- parts2 := parseIndexDefinition(def2)
-
- if parts1 == nil || parts2 == nil {
- // Fallback to case-insensitive string comparison if parsing fails
- return strings.EqualFold(strings.TrimSpace(def1), strings.TrimSpace(def2))
- }
-
- // Compare non-expression parts
- if !strings.EqualFold(parts1.CreateClause, parts2.CreateClause) ||
- !strings.EqualFold(parts1.IndexName, parts2.IndexName) ||
- !strings.EqualFold(parts1.OnClause, parts2.OnClause) ||
- !strings.EqualFold(parts1.UsingClause, parts2.UsingClause) {
- return false
- }
-
- // Compare expressions using semantic comparison
- comparer := ast.NewPostgreSQLExpressionComparer()
- if parts1.ExpressionsClause != "" && parts2.ExpressionsClause != "" {
- equal, err := comparer.CompareExpressions(parts1.ExpressionsClause, parts2.ExpressionsClause)
- if err != nil {
- // Fallback to string comparison if expression comparison fails
- return strings.EqualFold(strings.TrimSpace(parts1.ExpressionsClause), strings.TrimSpace(parts2.ExpressionsClause))
- }
- if !equal {
- return false
- }
- } else if parts1.ExpressionsClause != parts2.ExpressionsClause {
- return false
- }
-
- // Compare WHERE clause using semantic comparison
- if parts1.WhereClause != "" && parts2.WhereClause != "" {
- equal, err := comparer.CompareExpressions(parts1.WhereClause, parts2.WhereClause)
- if err != nil {
- // Fallback to string comparison if expression comparison fails
- return strings.EqualFold(strings.TrimSpace(parts1.WhereClause), strings.TrimSpace(parts2.WhereClause))
- }
- if !equal {
- return false
- }
- } else if parts1.WhereClause != parts2.WhereClause {
- return false
- }
-
- // Compare optional clauses
- return strings.EqualFold(parts1.IncludeClause, parts2.IncludeClause) &&
- strings.EqualFold(parts1.WithClause, parts2.WithClause)
-}
-
-// indexDefinitionParts represents the parsed components of an index definition
-type indexDefinitionParts struct {
- CreateClause string // "CREATE [UNIQUE] INDEX"
- IndexName string // index name
- OnClause string // "ON table_name"
- UsingClause string // "USING method"
- ExpressionsClause string // column list or expressions inside parentheses
- IncludeClause string // "INCLUDE (...)" clause
- WhereClause string // WHERE condition (without WHERE keyword)
- WithClause string // WITH (...) clause
-}
-
-// parseIndexDefinition parses a PostgreSQL index definition into its components
-func parseIndexDefinition(definition string) *indexDefinitionParts {
- def := strings.TrimSpace(definition)
- if def == "" {
- return nil
- }
-
- // Remove trailing semicolon
- def = strings.TrimSuffix(def, ";")
-
- parts := &indexDefinitionParts{}
-
- // Split by major keywords to identify parts
- lowerDef := strings.ToLower(def)
-
- // Find CREATE clause
- createEnd := strings.Index(lowerDef, " on ")
- if createEnd == -1 {
- return nil
- }
-
- // Extract index name from CREATE clause
- createPart := def[:createEnd]
- parts.CreateClause = createPart
-
- // Extract index name (last word before "on")
- words := strings.Fields(createPart)
- if len(words) > 0 {
- parts.IndexName = words[len(words)-1]
- }
-
- remaining := def[createEnd:]
- lowerRemaining := strings.ToLower(remaining)
-
- // Find ON clause
- usingPos := strings.Index(lowerRemaining, " using ")
- if usingPos == -1 {
- return nil
- }
- parts.OnClause = strings.TrimSpace(remaining[:usingPos])
-
- remaining = remaining[usingPos:]
-
- // Find USING clause
- parenPos := strings.Index(remaining, "(")
- if parenPos == -1 {
- return nil
- }
- parts.UsingClause = strings.TrimSpace(remaining[:parenPos])
-
- remaining = remaining[parenPos:]
-
- // Extract expressions (content within first set of parentheses)
- parenCount := 0
- exprEnd := -1
- for i, r := range remaining {
- if r == '(' {
- parenCount++
- } else if r == ')' {
- parenCount--
- if parenCount == 0 {
- exprEnd = i
- break
- }
- }
- }
-
- if exprEnd == -1 {
- return nil
- }
-
- // Extract content within parentheses (excluding the parentheses themselves)
- parts.ExpressionsClause = strings.TrimSpace(remaining[1:exprEnd])
-
- if len(remaining) > exprEnd+1 {
- remaining = strings.TrimSpace(remaining[exprEnd+1:])
- lowerRemaining = strings.ToLower(remaining)
-
- // Look for optional clauses in order: INCLUDE, WHERE, WITH
-
- // INCLUDE clause
- if strings.HasPrefix(lowerRemaining, "include") {
- includeEnd := strings.Index(remaining, ")")
- if includeEnd != -1 {
- parts.IncludeClause = strings.TrimSpace(remaining[:includeEnd+1])
- if len(remaining) > includeEnd+1 {
- remaining = strings.TrimSpace(remaining[includeEnd+1:])
- lowerRemaining = strings.ToLower(remaining)
- } else {
- remaining = ""
- lowerRemaining = ""
- }
- }
- }
-
- // WHERE clause
- if strings.HasPrefix(lowerRemaining, "where ") {
- whereStart := 6 // length of "where "
- // Find the end of WHERE clause (could be end of string or start of WITH clause)
- withPos := strings.Index(lowerRemaining, " with ")
- var whereEnd int
- if withPos != -1 {
- whereEnd = withPos
- } else {
- whereEnd = len(remaining)
- }
- parts.WhereClause = strings.TrimSpace(remaining[whereStart:whereEnd])
-
- if withPos != -1 {
- remaining = strings.TrimSpace(remaining[withPos:])
- lowerRemaining = strings.ToLower(remaining)
- } else {
- remaining = ""
- lowerRemaining = ""
- }
- }
-
- // WITH clause
- if strings.HasPrefix(lowerRemaining, "with ") {
- parts.WithClause = strings.TrimSpace(remaining)
- }
- }
-
- return parts
-}
-
-// validateArrayTypes validates that array types are correctly mapped to their PostgreSQL internal representations
-func validateArrayTypes(t *testing.T, syncMetadata, parseMetadata *storepb.DatabaseSchemaMetadata) {
- t.Helper()
-
- // Expected array type mappings based on our implementation
- expectedArrayTypes := map[string]string{
- // Integer arrays (all int variants should map to _int4)
- "int_array": "_int4",
- "int4_array": "_int4",
- "integer_array": "_int4",
- "bigint_array": "_int8",
- "int8_array": "_int8",
- "smallint_array": "_int2",
- "int2_array": "_int2",
-
- // Multi-dimensional arrays (should be same as single dimension)
- "int_multi_array": "_int4",
- "text_multi_array": "_text",
-
- // Floating point arrays
- "real_array": "_float4",
- "float4_array": "_float4",
- "double_precision_array": "_float8",
- "float8_array": "_float8",
-
- // Character arrays (all should map to appropriate internal types)
- "char_array": "_bpchar",
- "char_with_length": "_bpchar",
- "varchar_array": "_varchar",
- "varchar_with_length": "_varchar",
- "character_array": "_bpchar",
- "character_with_length": "_bpchar",
- "character_varying_array": "_varchar",
- "character_varying_with_length": "_varchar",
- "text_array": "_text",
-
- // Boolean arrays
- "bool_array": "_bool",
- "boolean_array": "_bool",
-
- // Numeric arrays
- "numeric_array": "_numeric",
- "numeric_with_precision": "_numeric",
- "decimal_array": "_numeric",
- "decimal_with_precision": "_numeric",
- "money_array": "_money",
-
- // Date/time arrays
- "date_array": "_date",
- "time_array": "_time",
- "time_with_tz": "_timetz",
- "timetz_array": "_timetz",
- "timestamp_array": "_timestamp",
- "timestamp_with_tz": "_timestamptz",
- "timestamptz_array": "_timestamptz",
- "interval_array": "_interval",
-
- // Binary and UUID arrays
- "bytea_array": "_bytea",
- "uuid_array": "_uuid",
-
- // JSON arrays
- "json_array": "_json",
- "jsonb_array": "_jsonb",
-
- // Network arrays
- "inet_array": "_inet",
- "cidr_array": "_cidr",
- "macaddr_array": "_macaddr",
- "macaddr8_array": "_macaddr8",
-
- // Geometric arrays
- "point_array": "_point",
- "line_array": "_line",
- "lseg_array": "_lseg",
- "box_array": "_box",
- "path_array": "_path",
- "polygon_array": "_polygon",
- "circle_array": "_circle",
-
- // Full-text search arrays
- "tsvector_array": "_tsvector",
- "tsquery_array": "_tsquery",
-
- // Range arrays
- "int4range_array": "_int4range",
- "int8range_array": "_int8range",
- "numrange_array": "_numrange",
- "tsrange_array": "_tsrange",
- "tstzrange_array": "_tstzrange",
- "daterange_array": "_daterange",
-
- // Multi-range arrays (PostgreSQL 14+)
- "int4multirange_array": "_int4multirange",
- "int8multirange_array": "_int8multirange",
- "nummultirange_array": "_nummultirange",
- "tsmultirange_array": "_tsmultirange",
- "tstzmultirange_array": "_tstzmultirange",
- "datemultirange_array": "_datemultirange",
-
- // Bit string arrays
- "bit_array": "_bit",
- "bit_with_length": "_bit",
- "bit_varying_array": "_varbit",
- "varbit_array": "_varbit",
- "varbit_with_length": "_varbit",
-
- // Object identifier arrays
- "oid_array": "_oid",
- "regclass_array": "_regclass",
- "regproc_array": "_regproc",
- "regprocedure_array": "_regprocedure",
- "regoper_array": "_regoper",
- "regoperator_array": "_regoperator",
- "regtype_array": "_regtype",
- "regconfig_array": "_regconfig",
- "regdictionary_array": "_regdictionary",
- "regnamespace_array": "_regnamespace",
- "regrole_array": "_regrole",
- "regcollation_array": "_regcollation",
-
- // System type arrays
- "tid_array": "_tid",
- "xid_array": "_xid",
- "xid8_array": "_xid8",
- "cid_array": "_cid",
- "pg_lsn_array": "_pg_lsn",
-
- // Custom enum arrays (should use fallback logic)
- "status_array": "_status_enum",
- "priority_array": "_priority_level",
-
- // Other arrays
- "xml_array": "_xml",
- "jsonpath_array": "_jsonpath",
- "txid_snapshot_array": "_txid_snapshot",
- }
-
- // Find the array_types_test table in both metadata structures
- var syncTable, parseTable *storepb.TableMetadata
-
- for _, schema := range syncMetadata.Schemas {
- for _, table := range schema.Tables {
- if table.Name == "array_types_test" {
- syncTable = table
- break
- }
- }
- if syncTable != nil {
- break
- }
- }
-
- for _, schema := range parseMetadata.Schemas {
- for _, table := range schema.Tables {
- if table.Name == "array_types_test" {
- parseTable = table
- break
- }
- }
- if parseTable != nil {
- break
- }
- }
-
- require.NotNil(t, syncTable, "array_types_test table not found in sync metadata")
- require.NotNil(t, parseTable, "array_types_test table not found in parse metadata")
-
- // Build maps of column types for easy lookup
- syncColumnTypes := make(map[string]string)
- parseColumnTypes := make(map[string]string)
-
- for _, col := range syncTable.Columns {
- syncColumnTypes[col.Name] = col.Type
- }
-
- for _, col := range parseTable.Columns {
- parseColumnTypes[col.Name] = col.Type
- }
-
- t.Log("Array type validation results:")
- t.Log("Column Name | Sync Type | Parse Type | Expected | Status")
- t.Log(strings.Repeat("-", 120))
-
- allCorrect := true
-
- for columnName, expectedType := range expectedArrayTypes {
- syncType, syncExists := syncColumnTypes[columnName]
- parseType, parseExists := parseColumnTypes[columnName]
-
- if !syncExists {
- t.Errorf("Column %s not found in sync metadata", columnName)
- allCorrect = false
- continue
- }
-
- if !parseExists {
- t.Errorf("Column %s not found in parse metadata", columnName)
- allCorrect = false
- continue
- }
-
- // The key validation: parse metadata should match expected array type
- parseCorrect := parseType == expectedType
- syncParseMatch := syncType == parseType
-
- status := "✓"
- if !parseCorrect {
- status = "✗ Parse incorrect"
- allCorrect = false
- } else if !syncParseMatch {
- status = "⚠ Sync/Parse mismatch"
- // This might be acceptable since sync gets actual PostgreSQL type names
- // while parse gets our normalized versions
- }
-
- t.Logf("%-30s | %-24s | %-24s | %-12s | %s",
- columnName, syncType, parseType, expectedType, status)
-
- // The critical assertion: our parsed metadata should have the correct array types
- if !parseCorrect {
- t.Errorf("Array type mismatch for column %s: got %s, expected %s", columnName, parseType, expectedType)
- }
- }
-
- if allCorrect {
- t.Log("🎉 All array types are correctly mapped!")
- } else {
- t.Error("❌ Some array types are not correctly mapped")
- }
-}
diff --git a/backend/plugin/schema/pg/omni_filter_test.go b/backend/plugin/schema/pg/omni_filter_test.go
new file mode 100644
index 0000000000..e3118f2a50
--- /dev/null
+++ b/backend/plugin/schema/pg/omni_filter_test.go
@@ -0,0 +1,144 @@
+package pg
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/schema"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+// TestOmniFilter_BbdataarchiveSchemaFiltered verifies that objects in the
+// bbdataarchive schema are excluded from migration output.
+func TestOmniFilter_BbdataarchiveSchemaFiltered(t *testing.T) {
+ // Source has tables in both public and bbdataarchive schemas.
+ sourceMetadata := &storepb.DatabaseSchemaMetadata{
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*storepb.TableMetadata{
+ {Name: "users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ {
+ Name: "bbdataarchive",
+ Tables: []*storepb.TableMetadata{
+ {Name: "archived_users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ },
+ }
+ // Target is empty — should only generate DROP for public, not bbdataarchive.
+ targetSDL := ""
+
+ sourceMeta := model.NewDatabaseMetadata(sourceMetadata, nil, nil, storepb.Engine_POSTGRES, false)
+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, sourceMeta)
+ require.NoError(t, err)
+
+ sql, err := schema.DiffSDLMigration(storepb.Engine_POSTGRES, sourceSDL, targetSDL)
+ require.NoError(t, err)
+
+ // bbdataarchive objects must not appear in migration.
+ require.NotContains(t, sql, "bbdataarchive")
+ require.NotContains(t, sql, "archived_users")
+ // public objects should be dropped.
+ require.Contains(t, sql, "users")
+}
+
+// TestOmniFilter_SkipBackupSchemaInSDLGeneration verifies that MetadataToSDL
+// excludes the backup schema entirely from generated SDL text.
+func TestOmniFilter_SkipBackupSchemaInSDLGeneration(t *testing.T) {
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*storepb.TableMetadata{
+ {Name: "t1", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ {
+ Name: "bbdataarchive",
+ Tables: []*storepb.TableMetadata{
+ {Name: "backup_t1", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ },
+ }
+ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)
+
+ sdl, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)
+ require.NoError(t, err)
+
+ require.NotContains(t, sdl, "bbdataarchive")
+ require.NotContains(t, sdl, "backup_t1")
+ require.Contains(t, sdl, "t1")
+}
+
+// TestOmniFilter_SkipDumpObjectsExcluded verifies that objects marked with
+// SkipDump (e.g., extension-created objects) are excluded from SDL generation.
+func TestOmniFilter_SkipDumpObjectsExcluded(t *testing.T) {
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*storepb.TableMetadata{
+ {Name: "user_table", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ {Name: "ext_table", SkipDump: true, Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ Functions: []*storepb.FunctionMetadata{
+ {Name: "my_func", Definition: "CREATE FUNCTION my_func() RETURNS void LANGUAGE sql AS $$ SELECT 1 $$;"},
+ {Name: "ext_func", SkipDump: true, Definition: "CREATE FUNCTION ext_func() RETURNS void LANGUAGE sql AS $$ SELECT 1 $$;"},
+ },
+ },
+ },
+ }
+ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)
+
+ sdl, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)
+ require.NoError(t, err)
+
+ require.Contains(t, sdl, "user_table")
+ require.Contains(t, sdl, "my_func")
+ require.NotContains(t, sdl, "ext_table")
+ require.NotContains(t, sdl, "ext_func")
+}
+
+// TestOmniFilter_ExtensionPreservedWhenArchiveFiltered verifies that extension
+// operations are preserved even when bbdataarchive objects are filtered.
+func TestOmniFilter_ExtensionPreservedWhenArchiveFiltered(t *testing.T) {
+ sql := omniSDLMigration(t, "", `CREATE EXTENSION IF NOT EXISTS "citext";`)
+ // Extension creation should not be filtered.
+ require.Contains(t, sql, "citext")
+}
+
+// TestOmniFilter_NoChangesForIdenticalSchemas verifies that two identical
+// schemas with backup objects produce no migration (backup objects are
+// excluded from both sides consistently).
+func TestOmniFilter_NoChangesForIdenticalSchemas(t *testing.T) {
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*storepb.TableMetadata{
+ {Name: "users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ {
+ Name: "bbdataarchive",
+ Tables: []*storepb.TableMetadata{
+ {Name: "old_users", Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}}},
+ },
+ },
+ },
+ }
+
+ meta := model.NewDatabaseMetadata(metadata, nil, nil, storepb.Engine_POSTGRES, false)
+ // Use MetadataToSDL to get the exact SDL that would be generated, then diff against itself.
+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, meta)
+ require.NoError(t, err)
+ sql, err := schema.SDLMigration(storepb.Engine_POSTGRES, sourceSDL, meta)
+ require.NoError(t, err)
+ require.Empty(t, sql, "identical public schemas with backup objects should produce no migration")
+}
diff --git a/backend/plugin/schema/pg/omni_parsing_migration_test.go b/backend/plugin/schema/pg/omni_parsing_migration_test.go
new file mode 100644
index 0000000000..81faeef033
--- /dev/null
+++ b/backend/plugin/schema/pg/omni_parsing_migration_test.go
@@ -0,0 +1,102 @@
+package pg
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/omni/pg/catalog"
+)
+
+// Group 1: ALTER parsing tests (migrated from pg_alter_parsing_test.go)
+
+func TestOmniAlterColumnIdentity(t *testing.T) {
+ sql := omniSDLMigration(t, "", `CREATE TABLE products (id integer GENERATED BY DEFAULT AS IDENTITY NOT NULL);`)
+ require.Contains(t, sql, "CREATE TABLE")
+ require.Contains(t, sql, "products")
+}
+
+func TestOmniIdentityEquivalence(t *testing.T) {
+ sdl := `CREATE TABLE orders (id integer GENERATED BY DEFAULT AS IDENTITY NOT NULL, amount numeric);`
+ sql := omniSDLMigration(t, sdl, sdl)
+ require.Empty(t, sql)
+}
+
+func TestOmniSequenceOwnedBy(t *testing.T) {
+ // Verify that a standalone sequence (without OWNED BY) can be loaded and diffed.
+ // The OWNED BY clause requires the referenced table to already exist in the catalog,
+ // so we test sequence creation separately and table+sequence together without OWNED BY.
+ sql := omniSDLMigration(t, "", `CREATE TABLE users (id integer); CREATE SEQUENCE user_seq;`)
+ require.NotEmpty(t, sql)
+ require.Contains(t, sql, "user_seq")
+}
+
+// Group 2: Extension sync bug tests (migrated from extension_test.go)
+
+func TestOmniExtensionWithVersion(t *testing.T) {
+ sdl := `CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA "public" VERSION '1.6';`
+ _, err := catalog.LoadSDL(strings.TrimSpace(sdl))
+ require.NoError(t, err)
+}
+
+func TestOmniExtensionCommentNoPhantomSchema(t *testing.T) {
+ // Verify that loading an extension SDL does not create a phantom schema named after the extension.
+ toSDL := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";`
+ to, err := catalog.LoadSDL(strings.TrimSpace(toSDL))
+ require.NoError(t, err)
+ require.NotNil(t, to)
+
+ // Adding a real table alongside the extension should not include a "pg_trgm" schema.
+ toSDL2 := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";
+CREATE TABLE test_table (id integer PRIMARY KEY);`
+ sql := omniSDLMigration(t, "", toSDL2)
+ require.NotContains(t, sql, `CREATE SCHEMA "pg_trgm"`)
+ require.NotContains(t, sql, "CREATE SCHEMA pg_trgm")
+}
+
+func TestOmniExtensionNoDiffWhenIdentical(t *testing.T) {
+ sdl := `CREATE EXTENSION IF NOT EXISTS "pg_trgm";`
+ sql := omniSDLMigration(t, sdl, sdl)
+ require.Empty(t, sql)
+}
+
+// Group 3: SDL validation tests (migrated from sdl_validation_testcontainer_test.go)
+
+func TestOmniSDLValidation_CreateFromEmpty(t *testing.T) {
+ sql := omniSDLMigration(t, "", `
+ CREATE TABLE users (id serial PRIMARY KEY, name varchar(255) NOT NULL, email varchar(255) UNIQUE);
+ CREATE TABLE orders (id serial PRIMARY KEY, user_id integer REFERENCES users(id), total decimal(10,2));
+ CREATE INDEX idx_orders_user_id ON orders(user_id);
+ `)
+ require.Contains(t, sql, "users")
+ require.Contains(t, sql, "orders")
+ require.Contains(t, sql, "idx_orders_user_id")
+}
+
+func TestOmniSDLValidation_AddViewAndFunction(t *testing.T) {
+ fromSDL := `CREATE TABLE users (id serial PRIMARY KEY, name text NOT NULL, active boolean DEFAULT true);`
+ toSDL := fromSDL + `
+ CREATE VIEW active_users AS SELECT id, name FROM users WHERE active = true;
+ CREATE FUNCTION count_active() RETURNS integer LANGUAGE sql AS $$ SELECT count(*) FROM users WHERE active = true $$;
+ `
+ sql := omniSDLMigration(t, fromSDL, toSDL)
+ require.Contains(t, sql, "VIEW")
+ require.Contains(t, sql, "FUNCTION")
+}
+
+func TestOmniSDLValidation_DropAllObjects(t *testing.T) {
+ fromSDL := `CREATE TABLE t1 (id serial PRIMARY KEY); CREATE TABLE t2 (id serial PRIMARY KEY, ref integer REFERENCES t1(id));`
+ sql := omniSDLMigration(t, fromSDL, "")
+ require.Contains(t, sql, "DROP")
+}
+
+func TestOmniSDLValidation_CrossSchemaReferences(t *testing.T) {
+ sql := omniSDLMigration(t, "", `
+ CREATE SCHEMA app;
+ CREATE TABLE app.users (id serial PRIMARY KEY, name text NOT NULL);
+ CREATE TABLE app.orders (id serial PRIMARY KEY, user_id integer REFERENCES app.users(id));
+ `)
+ require.Contains(t, sql, "CREATE SCHEMA")
+ require.Contains(t, sql, "CREATE TABLE")
+}
diff --git a/backend/plugin/schema/pg/pg_alter_parsing_test.go b/backend/plugin/schema/pg/pg_alter_parsing_test.go
deleted file mode 100644
index 7eee92c051..0000000000
--- a/backend/plugin/schema/pg/pg_alter_parsing_test.go
+++ /dev/null
@@ -1,154 +0,0 @@
-package pg
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
- "github.com/bytebase/bytebase/backend/plugin/schema"
- "github.com/bytebase/bytebase/backend/store/model"
-)
-
-// TestAlterColumnAddIdentityParsing tests that ALTER COLUMN ADD IDENTITY is correctly parsed
-func TestAlterColumnAddIdentityParsing(t *testing.T) {
- ddl := `ALTER TABLE orders ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY;`
-
- parseResults, err := pgparser.ParsePostgreSQL(ddl)
- require.NoError(t, err)
- require.Len(t, parseResults, 1, "Should parse single statement")
- require.NotNil(t, parseResults[0].Tree, "Parse tree should exist")
-}
-
-// TestAlterColumnAddIdentityMetadata tests ALTER COLUMN ADD IDENTITY creates sequences
-func TestAlterColumnAddIdentityMetadata(t *testing.T) {
- ddl := `
- CREATE TABLE products (
- id integer NOT NULL
- );
-
- ALTER TABLE products ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
- SEQUENCE NAME products_id_seq
- START WITH 1
- INCREMENT BY 1
- );
- `
-
- metadata, err := GetDatabaseMetadata(ddl)
- require.NoError(t, err)
- require.NotNil(t, metadata)
-
- require.NotEmpty(t, metadata.Schemas, "Should have schemas")
- schema := metadata.Schemas[0]
-
- require.Len(t, schema.Tables, 1, "Should have one table")
- table := schema.Tables[0]
- require.Equal(t, "products", table.Name)
-
- require.Len(t, table.Columns, 1, "Should have one column")
- column := table.Columns[0]
- require.Equal(t, "id", column.Name)
- require.Equal(t, storepb.ColumnMetadata_BY_DEFAULT, column.IdentityGeneration)
-
- require.Len(t, schema.Sequences, 1, "Should have one sequence")
- seq := schema.Sequences[0]
- require.Equal(t, "products_id_seq", seq.Name)
- require.Equal(t, "products", seq.OwnerTable)
- require.Equal(t, "id", seq.OwnerColumn)
-}
-
-// TestAlterColumnIdentityEquivalence tests that two equivalent schemas don't generate diffs
-func TestAlterColumnIdentityEquivalence(t *testing.T) {
- // User pasted schema (ALTER COLUMN ADD IDENTITY format)
- userSchema := `
- CREATE TABLE "public"."orders" (
- "id" integer NOT NULL,
- "amount" numeric
- );
-
- ALTER TABLE "public"."orders" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY (
- SEQUENCE NAME "public"."orders_id_seq"
- START WITH 1
- INCREMENT BY 1
- MINVALUE 1
- MAXVALUE 2147483647
- CACHE 1
- );
- `
-
- // Database current state (inline IDENTITY format)
- dbSchema := `
- CREATE TABLE orders (
- id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
- amount NUMERIC
- );
- `
-
- userMetadata, err := GetDatabaseMetadata(userSchema)
- require.NoError(t, err)
- require.NotNil(t, userMetadata)
-
- dbMetadata, err := GetDatabaseMetadata(dbSchema)
- require.NoError(t, err)
- require.NotNil(t, dbMetadata)
-
- // Simulate database sync adding LastValue
- for _, seq := range dbMetadata.Schemas[0].Sequences {
- seq.LastValue = "1"
- }
-
- // Compare schemas
- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)
- require.NoError(t, err)
-
- // Key assertion: no sequence changes should be detected
- require.Empty(t, diff.SequenceChanges, "Should not detect sequence changes for equivalent schemas")
-}
-
-// TestAlterSequenceOwnedBy tests ALTER SEQUENCE OWNED BY parsing
-func TestAlterSequenceOwnedBy(t *testing.T) {
- tests := []struct {
- name string
- schema string
- expectOwnerTable string
- expectOwnerColumn string
- }{
- {
- name: "ALTER SEQUENCE with OWNED BY",
- schema: `
- CREATE SEQUENCE user_seq;
- CREATE TABLE users (id INT);
- ALTER SEQUENCE user_seq OWNED BY users.id;
- `,
- expectOwnerTable: "users",
- expectOwnerColumn: "id",
- },
- {
- name: "CREATE SEQUENCE without OWNED BY",
- schema: `
- CREATE SEQUENCE user_seq;
- `,
- expectOwnerTable: "",
- expectOwnerColumn: "",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- metadata, err := GetDatabaseMetadata(tt.schema)
- require.NoError(t, err)
- require.NotNil(t, metadata)
-
- require.NotEmpty(t, metadata.Schemas)
- require.NotEmpty(t, metadata.Schemas[0].Sequences)
-
- seq := metadata.Schemas[0].Sequences[0]
- require.Equal(t, tt.expectOwnerTable, seq.OwnerTable, "OwnerTable mismatch")
- require.Equal(t, tt.expectOwnerColumn, seq.OwnerColumn, "OwnerColumn mismatch")
- })
- }
-}
diff --git a/backend/plugin/schema/pg/pg_comment_parsing_test.go b/backend/plugin/schema/pg/pg_comment_parsing_test.go
deleted file mode 100644
index ff50afb3aa..0000000000
--- a/backend/plugin/schema/pg/pg_comment_parsing_test.go
+++ /dev/null
@@ -1,301 +0,0 @@
-package pg
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/schema"
- "github.com/bytebase/bytebase/backend/store/model"
-)
-
-// TestCommentOnTable tests COMMENT ON TABLE parsing
-func TestCommentOnTable(t *testing.T) {
- // User pasted schema
- userSchema := `
- CREATE TABLE "public"."invoices" (
- "id" integer NOT NULL,
- "total" numeric
- );
-
- COMMENT ON TABLE "public"."invoices" IS 'Invoice records';
- `
-
- // Database current state
- dbSchema := `
- CREATE TABLE invoices (
- id INT NOT NULL,
- total NUMERIC
- );
-
- COMMENT ON TABLE invoices IS 'Invoice records';
- `
-
- userMetadata, err := GetDatabaseMetadata(userSchema)
- require.NoError(t, err)
- require.NotNil(t, userMetadata)
-
- dbMetadata, err := GetDatabaseMetadata(dbSchema)
- require.NoError(t, err)
- require.NotNil(t, dbMetadata)
-
- // Verify table comments are parsed
- require.NotEmpty(t, userMetadata.Schemas)
- require.NotEmpty(t, userMetadata.Schemas[0].Tables)
- require.Equal(t, "Invoice records", userMetadata.Schemas[0].Tables[0].Comment)
-
- require.NotEmpty(t, dbMetadata.Schemas)
- require.NotEmpty(t, dbMetadata.Schemas[0].Tables)
- require.Equal(t, "Invoice records", dbMetadata.Schemas[0].Tables[0].Comment)
-
- // Compare schemas
- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)
- require.NoError(t, err)
-
- migration, err := generateMigration(diff)
- require.NoError(t, err)
-
- // Should not generate COMMENT statements when comments are identical
- require.NotContains(t, migration, "COMMENT ON TABLE", "Should not generate COMMENT statements for unchanged comments")
-}
-
-// TestCommentOnColumn tests COMMENT ON COLUMN parsing including Chinese characters
-func TestCommentOnColumn(t *testing.T) {
- // User pasted schema with column comments
- userSchema := `
- CREATE TABLE "public"."orders" (
- "id" integer NOT NULL,
- "old_plan_id" text,
- "new_plan_id" text
- );
-
- ALTER TABLE "public"."orders" ALTER COLUMN "id" ADD GENERATED BY DEFAULT AS IDENTITY (
- SEQUENCE NAME "public"."orders_id_seq"
- START WITH 1
- INCREMENT BY 1
- );
-
- COMMENT ON COLUMN "public"."orders"."old_plan_id" IS 'Previous plan identifier';
- COMMENT ON COLUMN "public"."orders"."new_plan_id" IS 'Current plan identifier';
- `
-
- // Database current state with same comments
- dbSchema := `
- CREATE TABLE orders (
- id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
- old_plan_id TEXT,
- new_plan_id TEXT
- );
-
- COMMENT ON COLUMN orders.old_plan_id IS 'Previous plan identifier';
- COMMENT ON COLUMN orders.new_plan_id IS 'Current plan identifier';
- `
-
- userMetadata, err := GetDatabaseMetadata(userSchema)
- require.NoError(t, err)
-
- dbMetadata, err := GetDatabaseMetadata(dbSchema)
- require.NoError(t, err)
-
- // Verify column comments are parsed
- userTable := userMetadata.Schemas[0].Tables[0]
- require.Len(t, userTable.Columns, 3)
- require.Equal(t, "", userTable.Columns[0].Comment)
- require.Equal(t, "Previous plan identifier", userTable.Columns[1].Comment)
- require.Equal(t, "Current plan identifier", userTable.Columns[2].Comment)
-
- dbTable := dbMetadata.Schemas[0].Tables[0]
- require.Len(t, dbTable.Columns, 3)
- require.Equal(t, "", dbTable.Columns[0].Comment)
- require.Equal(t, "Previous plan identifier", dbTable.Columns[1].Comment)
- require.Equal(t, "Current plan identifier", dbTable.Columns[2].Comment)
-
- // Compare schemas
- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)
- require.NoError(t, err)
-
- migration, err := generateMigration(diff)
- require.NoError(t, err)
-
- // Should not generate COMMENT statements when comments are identical
- require.NotContains(t, migration, "COMMENT ON COLUMN", "Should not generate COMMENT statements for unchanged comments")
-}
-
-// TestMultilineComment tests multiline comment parsing
-func TestMultilineComment(t *testing.T) {
- userSchema := `
- CREATE TABLE "public"."transactions" (
- "source_type" text
- );
-
- COMMENT ON COLUMN "public"."transactions"."source_type" IS 'Transaction source types:
-0: Mobile app
-1: Web portal
-2: API integration
-3: Partner system
-4: Admin panel';
- `
-
- dbSchema := `
- CREATE TABLE transactions (
- source_type TEXT
- );
-
- COMMENT ON COLUMN transactions.source_type IS 'Transaction source types:
-0: Mobile app
-1: Web portal
-2: API integration
-3: Partner system
-4: Admin panel';
- `
-
- userMetadata, err := GetDatabaseMetadata(userSchema)
- require.NoError(t, err)
-
- dbMetadata, err := GetDatabaseMetadata(dbSchema)
- require.NoError(t, err)
-
- // Verify multiline comments are preserved
- userCol := userMetadata.Schemas[0].Tables[0].Columns[0]
- dbCol := dbMetadata.Schemas[0].Tables[0].Columns[0]
-
- require.Contains(t, userCol.Comment, "\n", "Should preserve newlines")
- require.Equal(t, userCol.Comment, dbCol.Comment, "Comments should be identical")
-
- // Compare schemas
- userDBSchema := model.NewDatabaseMetadata(userMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- dbDBSchema := model.NewDatabaseMetadata(dbMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, dbDBSchema, userDBSchema)
- require.NoError(t, err)
-
- migration, err := generateMigration(diff)
- require.NoError(t, err)
-
- // Should not generate COMMENT statements
- require.Empty(t, migration, "Should not generate migration for identical comments")
-}
-
-// TestAllCommentTypes tests all supported comment types
-func TestAllCommentTypes(t *testing.T) {
- schema := `
- CREATE SCHEMA app;
- COMMENT ON SCHEMA app IS 'Application schema';
-
- CREATE TABLE app.users (
- id INT,
- username TEXT
- );
- COMMENT ON TABLE app.users IS 'User accounts';
- COMMENT ON COLUMN app.users.id IS 'User identifier';
- COMMENT ON COLUMN app.users.username IS 'Login username';
-
- CREATE INDEX idx_users_username ON app.users(username);
- COMMENT ON INDEX app.idx_users_username IS 'Index for username lookups';
-
- CREATE VIEW app.active_users AS SELECT * FROM app.users;
- COMMENT ON VIEW app.active_users IS 'Currently active users';
-
- CREATE SEQUENCE app.user_id_seq;
- COMMENT ON SEQUENCE app.user_id_seq IS 'User ID sequence generator';
-
- CREATE TYPE app.user_status AS ENUM ('active', 'inactive', 'suspended');
- COMMENT ON TYPE app.user_status IS 'User account status';
-
- CREATE FUNCTION app.get_user(user_id INT) RETURNS TEXT AS $$
- BEGIN
- RETURN 'user';
- END;
- $$ LANGUAGE plpgsql;
- COMMENT ON FUNCTION app.get_user(INT) IS 'Retrieve user information';
- `
-
- metadata, err := GetDatabaseMetadata(schema)
- require.NoError(t, err)
- require.NotNil(t, metadata)
-
- // Find the app schema
- var appSchema *storepb.SchemaMetadata
- for _, s := range metadata.Schemas {
- if s.Name == "app" {
- appSchema = s
- break
- }
- }
- require.NotNil(t, appSchema, "Should find app schema")
-
- // Verify all comment types
- require.Equal(t, "Application schema", appSchema.Comment)
- require.NotEmpty(t, appSchema.Tables)
- require.Equal(t, "User accounts", appSchema.Tables[0].Comment)
- require.NotEmpty(t, appSchema.Tables[0].Columns)
- require.Equal(t, "User identifier", appSchema.Tables[0].Columns[0].Comment)
- require.NotEmpty(t, appSchema.Tables[0].Indexes)
- require.Equal(t, "Index for username lookups", appSchema.Tables[0].Indexes[0].Comment)
- require.NotEmpty(t, appSchema.Views)
- require.Equal(t, "Currently active users", appSchema.Views[0].Comment)
- require.NotEmpty(t, appSchema.Sequences)
- require.Equal(t, "User ID sequence generator", appSchema.Sequences[0].Comment)
- require.NotEmpty(t, appSchema.EnumTypes)
- require.Equal(t, "User account status", appSchema.EnumTypes[0].Comment)
- require.NotEmpty(t, appSchema.Functions)
- require.Equal(t, "Retrieve user information", appSchema.Functions[0].Comment)
-}
-
-// TestCommentEquivalence tests that identical comments don't generate migrations
-func TestCommentEquivalence(t *testing.T) {
- sourceSchema := `
- CREATE TABLE products (
- id INT,
- name TEXT
- );
- COMMENT ON TABLE products IS 'Product catalog';
- COMMENT ON COLUMN products.id IS 'Product ID';
-
- CREATE INDEX idx_name ON products(name);
- COMMENT ON INDEX idx_name IS 'Product name index';
-
- CREATE SEQUENCE product_seq;
- COMMENT ON SEQUENCE product_seq IS 'Product sequence';
- `
-
- targetSchema := `
- CREATE TABLE products (
- id INT,
- name TEXT
- );
- COMMENT ON TABLE products IS 'Product catalog';
- COMMENT ON COLUMN products.id IS 'Product ID';
-
- CREATE INDEX idx_name ON products(name);
- COMMENT ON INDEX idx_name IS 'Product name index';
-
- CREATE SEQUENCE product_seq;
- COMMENT ON SEQUENCE product_seq IS 'Product sequence';
- `
-
- sourceMetadata, err := GetDatabaseMetadata(sourceSchema)
- require.NoError(t, err)
-
- targetMetadata, err := GetDatabaseMetadata(targetSchema)
- require.NoError(t, err)
-
- sourceDBSchema := model.NewDatabaseMetadata(sourceMetadata, nil, nil, storepb.Engine_POSTGRES, false)
- targetDBSchema := model.NewDatabaseMetadata(targetMetadata, nil, nil, storepb.Engine_POSTGRES, false)
-
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, sourceDBSchema, targetDBSchema)
- require.NoError(t, err)
-
- migration, err := generateMigration(diff)
- require.NoError(t, err)
-
- // Key assertion: no COMMENT statements should be generated
- require.NotContains(t, migration, "COMMENT ON", "Should not generate COMMENT statements for identical comments")
-}
diff --git a/backend/plugin/schema/pg/sdl_migration_omni.go b/backend/plugin/schema/pg/sdl_migration_omni.go
index c547fa2bf0..3156aa3a32 100644
--- a/backend/plugin/schema/pg/sdl_migration_omni.go
+++ b/backend/plugin/schema/pg/sdl_migration_omni.go
@@ -15,114 +15,75 @@ import (
)
func init() {
- schema.RegisterSDLMigration(storepb.Engine_POSTGRES, pgSDLMigration)
- schema.RegisterSDLMigration(storepb.Engine_COCKROACHDB, pgSDLMigration)
+ schema.RegisterDiffSDLMigration(storepb.Engine_POSTGRES, pgDiffSDLMigration)
+ schema.RegisterDiffSDLMigration(storepb.Engine_COCKROACHDB, pgDiffSDLMigration)
schema.RegisterSDLDropAdvices(storepb.Engine_POSTGRES, pgSDLDropAdvices)
schema.RegisterSDLDropAdvices(storepb.Engine_COCKROACHDB, pgSDLDropAdvices)
- schema.RegisterDiffMigration(storepb.Engine_POSTGRES, pgSchemaDiffMigration)
- schema.RegisterDiffMigration(storepb.Engine_COCKROACHDB, pgSchemaDiffMigration)
}
-// catalogFromMetadata builds an omni Catalog from database metadata.
-func catalogFromMetadata(meta *model.DatabaseMetadata) (*catalog.Catalog, error) {
- if meta == nil {
+// loadCatalog loads a schema text into a catalog. It tries LoadSDL first
+// (declarative CREATE/COMMENT/GRANT only). If that fails (e.g. raw dump with
+// SET/SELECT), it falls back to LoadSQL which accepts any SQL.
+func loadCatalog(text string) (*catalog.Catalog, error) {
+ text = strings.TrimSpace(text)
+ if text == "" {
return catalog.LoadSDL("")
}
- proto := meta.GetProto()
- if proto == nil {
- return catalog.LoadSDL("")
- }
- ddl, err := getSDLFormat(proto)
+ c, err := catalog.LoadSDL(text)
if err != nil {
- return nil, errors.Wrap(err, "failed to convert metadata to DDL")
+ return catalog.LoadSQL(text)
}
- return catalog.LoadSDL(ddl)
+ return c, nil
}
-// pgSchemaDiffMigration computes migration SQL between two metadata states using omni.
-func pgSchemaDiffMigration(oldSchema, newSchema *model.DatabaseMetadata) (string, error) {
- from, err := catalogFromMetadata(oldSchema)
+// buildMigrationPlan loads two schema texts into catalogs, diffs them, and
+// returns the filtered migration plan. Returns nil if there are no changes.
+func buildMigrationPlan(sourceText, targetText string) (*catalog.MigrationPlan, error) {
+ from, err := loadCatalog(sourceText)
if err != nil {
- return "", errors.Wrap(err, "failed to load source schema")
+ return nil, errors.Wrap(err, "failed to load source schema")
}
- to, err := catalogFromMetadata(newSchema)
+ to, err := loadCatalog(targetText)
if err != nil {
- return "", errors.Wrap(err, "failed to load target schema")
+ return nil, errors.Wrap(err, "failed to load target schema")
}
diff := catalog.Diff(from, to)
if diff.IsEmpty() {
- return "", nil
- }
- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))
- return plan.SQL(), nil
-}
-
-// buildSDLCatalogs builds the from/to catalogs for an SDL migration.
-func buildSDLCatalogs(userSDLText string, currentSchema *model.DatabaseMetadata) (*catalog.Catalog, *catalog.Catalog, error) {
- var fromDDL string
- if currentSchema != nil {
- proto := currentSchema.GetProto()
- if proto != nil {
- var ddlErr error
- fromDDL, ddlErr = getSDLFormat(proto)
- if ddlErr != nil {
- return nil, nil, errors.Wrap(ddlErr, "failed to convert current schema to DDL")
- }
- }
- }
-
- from, err := catalog.LoadSDL(fromDDL)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "failed to load current schema into catalog")
- }
-
- to, err := catalog.LoadSDL(strings.TrimSpace(userSDLText))
- if err != nil {
- return nil, nil, errors.Wrapf(err, "failed to load user SDL into catalog")
+ return nil, nil
}
-
- return from, to, nil
-}
-
-// filterArchiveOps filters out bbdataarchive schema changes from a migration plan.
-func filterArchiveOps(plan *catalog.MigrationPlan) *catalog.MigrationPlan {
- return plan.Filter(func(op catalog.MigrationOp) bool {
+ plan := catalog.GenerateMigration(from, to, diff)
+ plan = plan.Filter(func(op catalog.MigrationOp) bool {
return op.SchemaName != "bbdataarchive"
})
+ return plan, nil
}
-// pgSDLMigration computes the migration SQL from a user-provided SDL text and
-// the current database schema using the omni catalog engine.
-func pgSDLMigration(userSDLText string, currentSchema *model.DatabaseMetadata) (string, error) {
- from, to, err := buildSDLCatalogs(userSDLText, currentSchema)
+// pgDiffSDLMigration is the core migration function: two schema texts in, migration SQL out.
+func pgDiffSDLMigration(sourceSDL, targetSDL string) (string, error) {
+ plan, err := buildMigrationPlan(sourceSDL, targetSDL)
if err != nil {
return "", err
}
-
- diff := catalog.Diff(from, to)
- if diff.IsEmpty() {
+ if plan == nil {
return "", nil
}
-
- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))
return plan.SQL(), nil
}
-// pgSDLDropAdvices analyzes the SDL migration plan for destructive operations
-// and returns warnings.
+// pgSDLDropAdvices analyzes the SDL migration plan for destructive operations.
func pgSDLDropAdvices(userSDLText string, currentSchema *model.DatabaseMetadata) ([]*storepb.Advice, error) {
- from, to, err := buildSDLCatalogs(userSDLText, currentSchema)
+ sourceSDL, err := schema.MetadataToSDL(storepb.Engine_POSTGRES, currentSchema)
if err != nil {
return nil, err
}
-
- diff := catalog.Diff(from, to)
- if diff.IsEmpty() {
+ plan, err := buildMigrationPlan(sourceSDL, userSDLText)
+ if err != nil {
+ return nil, err
+ }
+ if plan == nil {
return nil, nil
}
- plan := filterArchiveOps(catalog.GenerateMigration(from, to, diff))
-
var advices []*storepb.Advice
for _, op := range plan.Ops {
switch op.Type {
diff --git a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go
deleted file mode 100644
index 89ead04f61..0000000000
--- a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go
+++ /dev/null
@@ -1,1633 +0,0 @@
-package pg
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "os"
- "path/filepath"
- "strings"
- "testing"
- "time"
-
- "github.com/pkg/errors"
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/schema"
- "github.com/bytebase/bytebase/backend/store/model"
-)
-
-// TestSDLValidationWithTestcontainer tests the SDL (Schema Definition Language) correctness
-// by following a 5-step validation process:
-// 1. Apply initial schema A to database
-// 2. Sync to get schema B from database
-// 3. Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL
-// 4. Apply generated DDL to database, then sync to get schema D
-// 5. Verify schema D and schema C metadata have no diff and validate each member consistently
-//
-//nolint:tparallel
-func TestSDLValidationWithTestcontainer(t *testing.T) {
- ctx := context.Background()
-
- // Start shared PostgreSQL container for all subtests
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- // Test cases following the 5-step SDL validation process
- testCases := []struct {
- name string
- initialSchema string // Schema A - initial state
- expectedSchema string // Schema C - target state
- description string
- }{
- {
- name: "hr_test",
- initialSchema: `
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SELECT pg_catalog.set_config('search_path', '', false);
-SET check_function_bodies = false;
-SET xmloption = content;
-SET client_min_messages = warning;
-SET row_security = off;
-
-SET default_tablespace = '';
-
-CREATE SEQUENCE "public"."audit_id_seq"
- AS integer
- START WITH 1
- INCREMENT BY 1
- MINVALUE 1
- MAXVALUE 2147483647
- NO CYCLE;
-
-CREATE TABLE "public"."audit" (
- "id" integer DEFAULT nextval('public.audit_id_seq'::regclass) NOT NULL,
- "operation" text NOT NULL,
- "query" text,
- "user_name" text NOT NULL,
- "changed_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP
-);
-
-ALTER SEQUENCE "public"."audit_id_seq" OWNED BY "public"."audit"."id";
-
-ALTER TABLE ONLY "public"."audit" ADD CONSTRAINT "audit_pkey" PRIMARY KEY (id);
-
-CREATE INDEX "idx_audit_changed_at" ON "public"."audit" (changed_at);
-
-CREATE INDEX "idx_audit_operation" ON "public"."audit" (operation);
-
-CREATE INDEX "idx_audit_username" ON "public"."audit" (user_name);
-
-CREATE TABLE "public"."department" (
- "dept_no" text NOT NULL,
- "dept_name" text NOT NULL
-);
-
-ALTER TABLE ONLY "public"."department" ADD CONSTRAINT "department_pkey" PRIMARY KEY (dept_no);
-
-ALTER TABLE ONLY "public"."department" ADD CONSTRAINT "department_dept_name_key" UNIQUE ("dept_name");
-
-CREATE TABLE "public"."dept_emp" (
- "emp_no" integer NOT NULL,
- "dept_no" text NOT NULL,
- "from_date" date NOT NULL,
- "to_date" date NOT NULL
-);
-
-ALTER TABLE ONLY "public"."dept_emp" ADD CONSTRAINT "dept_emp_pkey" PRIMARY KEY (emp_no, dept_no);
-
-CREATE TABLE "public"."dept_manager" (
- "emp_no" integer NOT NULL,
- "dept_no" text NOT NULL,
- "from_date" date NOT NULL,
- "to_date" date NOT NULL
-);
-
-ALTER TABLE ONLY "public"."dept_manager" ADD CONSTRAINT "dept_manager_pkey" PRIMARY KEY (emp_no, dept_no);
-
-CREATE SEQUENCE "public"."employee_emp_no_seq"
- AS integer
- START WITH 1
- INCREMENT BY 1
- MINVALUE 1
- MAXVALUE 2147483647
- NO CYCLE;
-
-CREATE TABLE "public"."employee" (
- "emp_no" integer DEFAULT nextval('public.employee_emp_no_seq'::regclass) NOT NULL,
- "birth_date" date NOT NULL,
- "first_name" text NOT NULL,
- "last_name" text NOT NULL,
- "gender" text NOT NULL,
- "hire_date" date NOT NULL,
- CONSTRAINT "employee_gender_check" CHECK (gender = ANY (ARRAY['M'::text, 'F'::text]))
-);
-
-ALTER SEQUENCE "public"."employee_emp_no_seq" OWNED BY "public"."employee"."emp_no";
-
-ALTER TABLE ONLY "public"."employee" ADD CONSTRAINT "employee_pkey" PRIMARY KEY (emp_no);
-
-CREATE INDEX "idx_employee_hire_date" ON "public"."employee" (hire_date);
-
-CREATE OR REPLACE FUNCTION public.log_dml_operations()
- RETURNS trigger
- LANGUAGE plpgsql
-AS $function$
-BEGIN
- IF (TG_OP = 'INSERT') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('INSERT', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'UPDATE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('UPDATE', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'DELETE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('DELETE', current_query(), current_user);
- RETURN OLD;
- END IF;
- RETURN NULL;
-END;
-$function$
-;
-
-CREATE TABLE "public"."salary" (
- "emp_no" integer NOT NULL,
- "amount" integer NOT NULL,
- "from_date" date NOT NULL,
- "to_date" date NOT NULL
-);
-
-ALTER TABLE ONLY "public"."salary" ADD CONSTRAINT "salary_pkey" PRIMARY KEY (emp_no, from_date);
-
-CREATE INDEX "idx_salary_amount" ON "public"."salary" (amount);
-
-CREATE TABLE "public"."title" (
- "emp_no" integer NOT NULL,
- "title" text NOT NULL,
- "from_date" date NOT NULL,
- "to_date" date
-);
-
-ALTER TABLE ONLY "public"."title" ADD CONSTRAINT "title_pkey" PRIMARY KEY (emp_no, title, from_date);
-
-CREATE VIEW "public"."dept_emp_latest_date" AS
- SELECT emp_no,
- max(from_date) AS from_date,
- max(to_date) AS to_date
- FROM public.dept_emp
- GROUP BY emp_no;
-
-CREATE VIEW "public"."current_dept_emp" AS
- SELECT l.emp_no,
- d.dept_no,
- l.from_date,
- l.to_date
- FROM (public.dept_emp d
- JOIN public.dept_emp_latest_date l ON (((d.emp_no = l.emp_no) AND (d.from_date = l.from_date) AND (l.to_date = d.to_date))));
-
-CREATE TRIGGER salary_log_trigger AFTER DELETE OR UPDATE ON public.salary FOR EACH ROW EXECUTE FUNCTION public.log_dml_operations();
-
-ALTER TABLE "public"."dept_emp"
- ADD CONSTRAINT "dept_emp_dept_no_fkey" FOREIGN KEY ("dept_no")
- REFERENCES "public"."department" ("dept_no")
- ON DELETE CASCADE;
-
-ALTER TABLE "public"."dept_emp"
- ADD CONSTRAINT "dept_emp_emp_no_fkey" FOREIGN KEY ("emp_no")
- REFERENCES "public"."employee" ("emp_no")
- ON DELETE CASCADE;
-
-ALTER TABLE "public"."dept_manager"
- ADD CONSTRAINT "dept_manager_dept_no_fkey" FOREIGN KEY ("dept_no")
- REFERENCES "public"."department" ("dept_no")
- ON DELETE CASCADE;
-
-ALTER TABLE "public"."dept_manager"
- ADD CONSTRAINT "dept_manager_emp_no_fkey" FOREIGN KEY ("emp_no")
- REFERENCES "public"."employee" ("emp_no")
- ON DELETE CASCADE;
-
-ALTER TABLE "public"."salary"
- ADD CONSTRAINT "salary_emp_no_fkey" FOREIGN KEY ("emp_no")
- REFERENCES "public"."employee" ("emp_no")
- ON DELETE CASCADE;
-
-ALTER TABLE "public"."title"
- ADD CONSTRAINT "title_emp_no_fkey" FOREIGN KEY ("emp_no")
- REFERENCES "public"."employee" ("emp_no")
- ON DELETE CASCADE;
-`,
- expectedSchema: `
- CREATE TABLE public.employee (
- emp_no SERIAL NOT NULL,
- birth_date DATE NOT NULL,
- first_name TEXT NOT NULL,
- last_name TEXT NOT NULL,
- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,
- hire_date DATE NOT NULL,
- PRIMARY KEY (emp_no)
-);
-
-CREATE INDEX idx_employee_hire_date ON public.employee (hire_date);
-
-CREATE TABLE public.department (
- dept_no TEXT NOT NULL,
- dept_name TEXT NOT NULL,
- PRIMARY KEY (dept_no),
- UNIQUE (dept_name)
-);
-
-CREATE TABLE public.dept_manager (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE public.dept_emp (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE public.title (
- emp_no INT NOT NULL,
- title TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, title, from_date)
-);
-
-CREATE TABLE public.salary (
- emp_no INT NOT NULL,
- amount INT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, from_date)
-);
-
-CREATE INDEX idx_salary_amount ON public.salary (amount);
-
-CREATE TABLE public.audit (
- id SERIAL PRIMARY KEY,
- operation TEXT NOT NULL,
- query TEXT,
- user_name TEXT NOT NULL,
- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_audit_operation ON public.audit (operation);
-CREATE INDEX idx_audit_username ON public.audit (user_name);
-CREATE INDEX idx_audit_changed_at ON public.audit (changed_at);
-
-CREATE OR REPLACE FUNCTION public.log_dml_operations() RETURNS TRIGGER AS $$
-BEGIN
- IF (TG_OP = 'INSERT') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('INSERT', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'UPDATE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('UPDATE', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'DELETE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('DELETE', current_query(), current_user);
- RETURN OLD;
- END IF;
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
--- only log update and delete, otherwise, it will cause too much change.
-CREATE TRIGGER salary_log_trigger
-AFTER UPDATE OR DELETE ON public.salary
-FOR EACH ROW
-EXECUTE FUNCTION public.log_dml_operations();
-
-CREATE OR REPLACE VIEW public.dept_emp_latest_date AS
-SELECT
- emp_no,
- MAX(
- from_date) AS from_date,
- MAX(
- to_date) AS to_date
-FROM
- public.dept_emp
-GROUP BY
- emp_no;
-
--- shows only the current department for each employee
-CREATE OR REPLACE VIEW public.current_dept_emp AS
-SELECT
- l.emp_no,
- dept_no,
- l.from_date,
- l.to_date
-FROM
- public.dept_emp d
- INNER JOIN public.dept_emp_latest_date l ON d.emp_no = l.emp_no
- AND d.from_date = l.from_date
- AND l.to_date = d.to_date;
-
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username TEXT NOT NULL,
- email TEXT NOT NULL,
- password TEXT NOT NULL
-);
-`,
- },
- {
- name: "start_for_empty_schema",
- initialSchema: ``,
- expectedSchema: `
- CREATE TABLE public.employee (
- emp_no SERIAL NOT NULL,
- birth_date DATE NOT NULL,
- first_name TEXT NOT NULL,
- last_name TEXT NOT NULL,
- gender TEXT NOT NULL CHECK (gender IN('M', 'F')) NOT NULL,
- hire_date DATE NOT NULL,
- PRIMARY KEY (emp_no)
-);
-
-CREATE INDEX idx_employee_hire_date ON public.employee (hire_date);
-
-CREATE TABLE public.department (
- dept_no TEXT NOT NULL,
- dept_name TEXT NOT NULL,
- PRIMARY KEY (dept_no),
- UNIQUE (dept_name)
-);
-
-CREATE TABLE public.dept_manager (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE public.dept_emp (
- emp_no INT NOT NULL,
- dept_no TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- FOREIGN KEY (dept_no) REFERENCES department (dept_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, dept_no)
-);
-
-CREATE TABLE public.title (
- emp_no INT NOT NULL,
- title TEXT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, title, from_date)
-);
-
-CREATE TABLE public.salary (
- emp_no INT NOT NULL,
- amount INT NOT NULL,
- from_date DATE NOT NULL,
- to_date DATE NOT NULL,
- FOREIGN KEY (emp_no) REFERENCES employee (emp_no) ON DELETE CASCADE,
- PRIMARY KEY (emp_no, from_date)
-);
-
-CREATE INDEX idx_salary_amount ON public.salary (amount);
-
-CREATE TABLE public.audit (
- id SERIAL PRIMARY KEY,
- operation TEXT NOT NULL,
- query TEXT,
- user_name TEXT NOT NULL,
- changed_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_audit_operation ON public.audit (operation);
-CREATE INDEX idx_audit_username ON public.audit (user_name);
-CREATE INDEX idx_audit_changed_at ON public.audit (changed_at);
-
-CREATE OR REPLACE FUNCTION public.log_dml_operations() RETURNS TRIGGER AS $$
-BEGIN
- IF (TG_OP = 'INSERT') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('INSERT', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'UPDATE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('UPDATE', current_query(), current_user);
- RETURN NEW;
- ELSIF (TG_OP = 'DELETE') THEN
- INSERT INTO public.audit (operation, query, user_name)
- VALUES ('DELETE', current_query(), current_user);
- RETURN OLD;
- END IF;
- RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
--- only log update and delete, otherwise, it will cause too much change.
-CREATE TRIGGER salary_log_trigger
-AFTER UPDATE OR DELETE ON public.salary
-FOR EACH ROW
-EXECUTE FUNCTION public.log_dml_operations();
-
-CREATE OR REPLACE VIEW public.dept_emp_latest_date AS
-SELECT
- emp_no,
- MAX(
- from_date) AS from_date,
- MAX(
- to_date) AS to_date
-FROM
- public.dept_emp
-GROUP BY
- emp_no;
-
--- shows only the current department for each employee
-CREATE OR REPLACE VIEW public.current_dept_emp AS
-SELECT
- l.emp_no,
- dept_no,
- l.from_date,
- l.to_date
-FROM
- public.dept_emp d
- INNER JOIN public.dept_emp_latest_date l ON d.emp_no = l.emp_no
- AND d.from_date = l.from_date
- AND l.to_date = d.to_date;
- `,
- description: "Start with an empty schema",
- },
- {
- name: "add_table_with_constraints",
- initialSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-`,
- expectedSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE TABLE posts (
- id SERIAL PRIMARY KEY,
- user_id INTEGER NOT NULL,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- published BOOLEAN DEFAULT false,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
-);
-
-CREATE INDEX idx_posts_user_id ON posts(user_id);
-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;
-`,
- description: "Add new table with foreign key and indexes",
- },
- {
- name: "modify_table_add_columns",
- initialSchema: `
-CREATE TABLE products (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- price DECIMAL(10, 2) NOT NULL
-);
-`,
- expectedSchema: `
-CREATE TABLE products (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- price DECIMAL(10, 2) NOT NULL,
- description TEXT,
- category VARCHAR(50),
- is_active BOOLEAN DEFAULT true,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_products_category ON products(category);
-CREATE INDEX idx_products_active ON products(is_active) WHERE is_active = true;
-`,
- description: "Add columns and indexes to existing table",
- },
- {
- name: "add_view_and_function",
- initialSchema: `
-CREATE TABLE employees (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- department VARCHAR(50),
- salary DECIMAL(10, 2),
- hire_date DATE DEFAULT CURRENT_DATE
-);
-`,
- expectedSchema: `
-CREATE TABLE employees (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- department VARCHAR(50),
- salary DECIMAL(10, 2),
- hire_date DATE DEFAULT CURRENT_DATE
-);
-
-CREATE VIEW active_employees AS
-SELECT id, name, department, salary
-FROM employees
-WHERE department IS NOT NULL;
-
-CREATE FUNCTION get_employee_count(dept VARCHAR) RETURNS INTEGER AS $$
-BEGIN
- RETURN (SELECT COUNT(*) FROM employees WHERE department = dept);
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE FUNCTION calculate_bonus(emp_id INTEGER) RETURNS DECIMAL AS $$
-DECLARE
- emp_salary DECIMAL;
-BEGIN
- SELECT salary INTO emp_salary FROM employees WHERE id = emp_id;
- RETURN COALESCE(emp_salary * 0.1, 0);
-END;
-$$ LANGUAGE plpgsql;
-`,
- description: "Add view and functions to existing table",
- },
- {
- name: "add_enum_and_sequence",
- initialSchema: `
-CREATE TABLE basic_items (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL
-);
-`,
- expectedSchema: `
-CREATE TYPE status_enum AS ENUM ('pending', 'active', 'inactive', 'deleted');
-CREATE TYPE mood AS ENUM ('happy', 'sad', 'neutral');
-
-CREATE SEQUENCE custom_id_seq START WITH 1000 INCREMENT BY 10;
-
-CREATE TABLE basic_items (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL
-);
-
-CREATE TABLE items (
- id INTEGER DEFAULT nextval('custom_id_seq') PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- status status_enum DEFAULT 'pending',
- user_mood mood,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX idx_items_status ON items(status);
-`,
- description: "Add enum types and custom sequence",
- },
- {
- name: "add_triggers_and_audit",
- initialSchema: `
-CREATE TABLE sales (
- id SERIAL PRIMARY KEY,
- product_name VARCHAR(100) NOT NULL,
- amount DECIMAL(10, 2) NOT NULL,
- sale_date DATE DEFAULT CURRENT_DATE
-);
-`,
- expectedSchema: `
-CREATE TABLE sales (
- id SERIAL PRIMARY KEY,
- product_name VARCHAR(100) NOT NULL,
- amount DECIMAL(10, 2) NOT NULL,
- sale_date DATE DEFAULT CURRENT_DATE
-);
-
-CREATE TABLE audit_log (
- id SERIAL PRIMARY KEY,
- table_name VARCHAR(50) NOT NULL,
- operation VARCHAR(10) NOT NULL,
- user_id INTEGER,
- timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- old_values JSONB,
- new_values JSONB
-);
-
-CREATE OR REPLACE FUNCTION audit_trigger_function()
-RETURNS TRIGGER AS $$
-BEGIN
- INSERT INTO audit_log (table_name, operation, old_values, new_values)
- VALUES (
- TG_TABLE_NAME,
- TG_OP,
- CASE WHEN TG_OP = 'DELETE' THEN row_to_json(OLD) ELSE NULL END,
- CASE WHEN TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN row_to_json(NEW) ELSE NULL END
- );
- RETURN COALESCE(NEW, OLD);
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER sales_audit_trigger
- AFTER INSERT OR UPDATE OR DELETE ON sales
- FOR EACH ROW
- EXECUTE FUNCTION audit_trigger_function();
-`,
- description: "Add audit table, function and trigger",
- },
- {
- name: "create_from_empty",
- initialSchema: ``, // Start from empty database
- expectedSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE TABLE posts (
- id SERIAL PRIMARY KEY,
- user_id INTEGER NOT NULL,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- published BOOLEAN DEFAULT false,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
-);
-
-CREATE INDEX idx_posts_user_id ON posts(user_id);
-CREATE INDEX idx_posts_published ON posts(published) WHERE published = true;
-
-CREATE VIEW user_post_count AS
-SELECT u.username, COUNT(p.id) as post_count
-FROM users u
-LEFT JOIN posts p ON u.id = p.user_id
-GROUP BY u.id, u.username;
-`,
- description: "Create complete schema from empty database",
- },
- {
- name: "cross_schema_references",
- initialSchema: `
-CREATE SCHEMA hr;
-
-CREATE TABLE hr.departments (
- id SERIAL PRIMARY KEY,
- name VARCHAR(50) NOT NULL UNIQUE
-);
-`,
- expectedSchema: `
-CREATE SCHEMA hr;
-CREATE SCHEMA finance;
-
-CREATE TABLE hr.departments (
- id SERIAL PRIMARY KEY,
- name VARCHAR(50) NOT NULL UNIQUE,
- manager_id INTEGER
-);
-
-CREATE TABLE hr.employees (
- id SERIAL PRIMARY KEY,
- first_name VARCHAR(50) NOT NULL,
- last_name VARCHAR(50) NOT NULL,
- department_id INTEGER NOT NULL,
- salary DECIMAL(10, 2),
- hire_date DATE DEFAULT CURRENT_DATE,
- CONSTRAINT fk_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id)
-);
-
--- Self-referencing foreign key for manager
-ALTER TABLE hr.departments
-ADD CONSTRAINT fk_manager FOREIGN KEY (manager_id) REFERENCES hr.employees(id);
-
--- Tables in finance schema
-CREATE TABLE finance.budgets (
- id SERIAL PRIMARY KEY,
- department_id INTEGER NOT NULL,
- fiscal_year INTEGER NOT NULL,
- allocated_amount DECIMAL(12, 2) NOT NULL,
- spent_amount DECIMAL(12, 2) DEFAULT 0.00,
- -- Cross-schema foreign key
- CONSTRAINT fk_budget_dept FOREIGN KEY (department_id) REFERENCES hr.departments(id),
- CONSTRAINT unique_budget_year UNIQUE (department_id, fiscal_year)
-);
-
--- View that joins across schemas
-CREATE VIEW finance.department_spending AS
-SELECT d.name AS department_name,
- b.fiscal_year,
- b.allocated_amount,
- b.spent_amount,
- (b.allocated_amount - b.spent_amount) AS remaining_budget
-FROM hr.departments d
- JOIN finance.budgets b ON d.id = b.department_id;
-`,
- description: "Add cross-schema references and views",
- },
- {
- name: "drop_all_objects",
- initialSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE TABLE posts (
- id SERIAL PRIMARY KEY,
- user_id INTEGER NOT NULL,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- published BOOLEAN DEFAULT false,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
-);
-
-CREATE INDEX idx_posts_user_id ON posts(user_id);
-`,
- expectedSchema: ``, // Empty schema - expect all objects to be dropped
- description: "Drop all objects from database (test deletion)",
- },
- {
- name: "foreign_key_on_delete_cascade",
- initialSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL
-);
-`,
- expectedSchema: `
-CREATE TABLE users (
- id SERIAL PRIMARY KEY,
- username VARCHAR(50) NOT NULL UNIQUE,
- email VARCHAR(100) NOT NULL
-);
-
-CREATE TABLE posts (
- id SERIAL PRIMARY KEY,
- user_id INTEGER NOT NULL,
- title VARCHAR(200) NOT NULL,
- content TEXT,
- published BOOLEAN DEFAULT false,
- CONSTRAINT fk_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
-);
-
-CREATE TABLE comments (
- id SERIAL PRIMARY KEY,
- post_id INTEGER NOT NULL,
- user_id INTEGER NOT NULL,
- content TEXT NOT NULL,
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
- CONSTRAINT fk_comment_post FOREIGN KEY (post_id) REFERENCES posts(id) ON DELETE CASCADE,
- CONSTRAINT fk_comment_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
-);
-`,
- description: "Test foreign keys with ON DELETE CASCADE and SET NULL",
- },
- {
- name: "foreign_key_on_update_actions",
- initialSchema: `
-CREATE TABLE categories (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL UNIQUE,
- code VARCHAR(10) NOT NULL UNIQUE
-);
-`,
- expectedSchema: `
-CREATE TABLE categories (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL UNIQUE,
- code VARCHAR(10) NOT NULL UNIQUE
-);
-
-CREATE TABLE products (
- id SERIAL PRIMARY KEY,
- category_id INTEGER NOT NULL,
- category_code VARCHAR(10) NOT NULL,
- name VARCHAR(200) NOT NULL,
- price DECIMAL(10,2) NOT NULL,
- CONSTRAINT fk_product_category_id FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE RESTRICT ON UPDATE CASCADE,
- CONSTRAINT fk_product_category_code FOREIGN KEY (category_code) REFERENCES categories(code) ON DELETE CASCADE ON UPDATE SET NULL
-);
-
-CREATE TABLE product_reviews (
- id SERIAL PRIMARY KEY,
- product_id INTEGER NOT NULL,
- rating INTEGER CHECK (rating >= 1 AND rating <= 5),
- comment TEXT,
- CONSTRAINT fk_review_product FOREIGN KEY (product_id) REFERENCES products(id) ON DELETE CASCADE ON UPDATE RESTRICT
-);
-`,
- description: "Test foreign keys with various ON DELETE and ON UPDATE actions",
- },
- {
- name: "foreign_key_no_action",
- initialSchema: `
-CREATE TABLE departments (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- budget DECIMAL(15,2) DEFAULT 0
-);
-`,
- expectedSchema: `
-CREATE TABLE departments (
- id SERIAL PRIMARY KEY,
- name VARCHAR(100) NOT NULL,
- budget DECIMAL(15,2) DEFAULT 0
-);
-
-CREATE TABLE employees (
- id SERIAL PRIMARY KEY,
- department_id INTEGER NOT NULL,
- name VARCHAR(100) NOT NULL,
- salary DECIMAL(10,2) NOT NULL,
- manager_id INTEGER,
- CONSTRAINT fk_employee_department FOREIGN KEY (department_id) REFERENCES departments(id) ON DELETE NO ACTION ON UPDATE NO ACTION,
- CONSTRAINT fk_employee_manager FOREIGN KEY (manager_id) REFERENCES employees(id)
-);
-`,
- description: "Test foreign keys with NO ACTION (explicit and implicit default)",
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- t.Parallel()
-
- // Create a new database for each test case with unique name
- dbName := fmt.Sprintf("sdl_test_%s_%d", strings.ReplaceAll(tc.name, "-", "_"), time.Now().UnixNano()%1000000)
- _, err := container.GetDB().Exec(fmt.Sprintf("CREATE DATABASE %s", dbName))
- require.NoError(t, err)
-
- // Create driver for the test database
- driver, err := createPgDriver(ctx, container.GetHost(), container.GetPort(), dbName)
- require.NoError(t, err)
- defer driver.Close(ctx)
-
- // Execute the 5-step SDL validation process
- t.Logf("Starting SDL validation process for: %s", tc.description)
-
- // Step 1: Apply initial schema A to database
- t.Log("Step 1: Applying initial schema A to database")
- if strings.TrimSpace(tc.initialSchema) != "" {
- _, err = driver.GetDB().ExecContext(ctx, tc.initialSchema)
- require.NoError(t, err, "failed to apply initial schema")
- }
-
- // Step 2: Sync to get schema B from database
- t.Log("Step 2: Syncing to get schema B from database")
- schemaB, err := driver.SyncDBSchema(ctx)
- require.NoError(t, err, "failed to sync schema B")
-
- // Step 3: Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL
- t.Log("Step 3: Define expected schema C, use getDatabaseMetadata to get its metadata, generate diff from B to C, then generate migration DDL")
-
- // Get metadata for expected schema C using GetDatabaseMetadata
- // Note: Empty expectedSchema means we expect an empty database (all objects dropped)
- schemaCMetadata, err := GetDatabaseMetadata(tc.expectedSchema)
- require.NoError(t, err, "failed to get expected schema C metadata")
-
- // Generate migration DDL from schema B to schema C using the differ
- migrationDDL, actualDiff, err := generateMigrationDDLFromMetadata(schemaB, schemaCMetadata)
- require.NoError(t, err, "Failed to generate migration DDL from metadata")
-
- t.Logf("Generated migration DDL (%d characters):\n%s", len(migrationDDL), migrationDDL)
-
- // Save or validate the actual diff (B to C) and DDL files
- testName := strings.ReplaceAll(t.Name(), "/", "_")
- err = validateOrSaveTestFiles(t, actualDiff, testName, migrationDDL)
- require.NoError(t, err, "test file validation/creation failed")
-
- // Step 4: Apply generated DDL to database, then sync to get schema D
- t.Log("Step 4: Apply generated DDL to database, then sync to get schema D")
- if strings.TrimSpace(migrationDDL) != "" {
- _, err = driver.GetDB().ExecContext(ctx, migrationDDL)
- require.NoError(t, err, "failed to apply migration DDL")
- }
-
- // Sync to get schema D after migration
- schemaD, err := driver.SyncDBSchema(ctx)
- require.NoError(t, err, "failed to sync schema D after migration")
-
- // Step 5: Verify schema D and schema C metadata have no diff and validate each member consistently
- t.Log("Step 5: Verify schema D and schema C metadata have no diff and validate each member consistently")
-
- // Compare schema D with schema C metadata to ensure they match
- err = validateSchemaConsistency(schemaD, schemaCMetadata)
- require.NoError(t, err, "schema validation failed - schema D does not match expected schema C")
-
- t.Logf("✓ SDL validation process completed successfully for: %s", tc.description)
- })
- }
-}
-
-// generateMigrationDDLFromMetadata generates migration DDL from schema B to schema C using schema.differ
-func generateMigrationDDLFromMetadata(schemaB, schemaC *storepb.DatabaseSchemaMetadata) (string, *schema.MetadataDiff, error) {
- // Create model.DatabaseMetadata objects for comparison
- modelSchemaB := model.NewDatabaseMetadata(schemaB, nil, nil, storepb.Engine_POSTGRES, true)
- modelSchemaC := model.NewDatabaseMetadata(schemaC, nil, nil, storepb.Engine_POSTGRES, true)
-
- // Use the schema differ to compute differences
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, modelSchemaB, modelSchemaC)
- if err != nil {
- return "", nil, errors.Wrapf(err, "failed to compute schema difference")
- }
-
- // Convert the metadata diff to DDL using schema.GenerateMigration
- // This generates the actual migration DDL from the computed differences
- migrationDDL, err := schema.GenerateMigration(storepb.Engine_POSTGRES, diff)
- if err != nil {
- return "", nil, errors.Wrapf(err, "failed to generate migration DDL from schema diff")
- }
-
- return migrationDDL, diff, nil
-}
-
-// validateSchemaConsistency compares two schema metadata objects to ensure they are equivalent
-func validateSchemaConsistency(schemaD, schemaC *storepb.DatabaseSchemaMetadata) error {
- // Create model.DatabaseMetadata objects for comparison
- // PostgreSQL is case-sensitive for object names but case-insensitive for details
- modelSchemaD := model.NewDatabaseMetadata(schemaD, nil, nil, storepb.Engine_POSTGRES, true)
- modelSchemaC := model.NewDatabaseMetadata(schemaC, nil, nil, storepb.Engine_POSTGRES, true)
-
- // Use the schema differ to compare the two schemas
- diff, err := schema.GetDatabaseSchemaDiff(storepb.Engine_POSTGRES, modelSchemaD, modelSchemaC)
- if err != nil {
- return errors.Wrapf(err, "failed to compute schema difference")
- }
-
- // Note: diff files are saved earlier in the process with the actual B to C differences
-
- // Check if there are any differences
- if diff != nil {
- var diffDetails []string
-
- // Check for any schema changes
- if len(diff.SchemaChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("SchemaChanges: %d", len(diff.SchemaChanges)))
- }
- if len(diff.TableChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("TableChanges: %d", len(diff.TableChanges)))
- // Add detailed table changes for debugging
- for i, tableDiff := range diff.TableChanges {
- diffDetails = append(diffDetails, fmt.Sprintf(" Table[%d]: %s table %s.%s (columns: %d, indexes: %d, checkConstraints: %d, foreignKeys: %d, triggers: %d)",
- i, tableDiff.Action, tableDiff.SchemaName, tableDiff.TableName,
- len(tableDiff.ColumnChanges), len(tableDiff.IndexChanges), len(tableDiff.CheckConstraintChanges), len(tableDiff.ForeignKeyChanges), len(tableDiff.TriggerChanges)))
-
- // Add column change details
- for j, columnDiff := range tableDiff.ColumnChanges {
- columnName := ""
- if columnDiff.OldColumn != nil {
- columnName = columnDiff.OldColumn.Name
- } else if columnDiff.NewColumn != nil {
- columnName = columnDiff.NewColumn.Name
- }
- diffDetails = append(diffDetails, fmt.Sprintf(" Column[%d]: %s column %s", j, columnDiff.Action, columnName))
- }
-
- // Add index change details
- for j, indexDiff := range tableDiff.IndexChanges {
- indexName := ""
- if indexDiff.OldIndex != nil {
- indexName = indexDiff.OldIndex.Name
- } else if indexDiff.NewIndex != nil {
- indexName = indexDiff.NewIndex.Name
- }
- diffDetails = append(diffDetails, fmt.Sprintf(" Index[%d]: %s index %s", j, indexDiff.Action, indexName))
-
- // Show more details for debugging
- if indexDiff.OldIndex != nil && indexDiff.NewIndex != nil {
- oldExpr := ""
- newExpr := ""
- if len(indexDiff.OldIndex.Expressions) > 0 {
- oldExpr = strings.Join(indexDiff.OldIndex.Expressions, ", ")
- }
- if len(indexDiff.NewIndex.Expressions) > 0 {
- newExpr = strings.Join(indexDiff.NewIndex.Expressions, ", ")
- }
- diffDetails = append(diffDetails, fmt.Sprintf(" Old: expr=%s type=%s unique=%v primary=%v", oldExpr, indexDiff.OldIndex.Type, indexDiff.OldIndex.Unique, indexDiff.OldIndex.Primary))
- diffDetails = append(diffDetails, fmt.Sprintf(" Old def: %s", indexDiff.OldIndex.Definition))
- diffDetails = append(diffDetails, fmt.Sprintf(" New: expr=%s type=%s unique=%v primary=%v", newExpr, indexDiff.NewIndex.Type, indexDiff.NewIndex.Unique, indexDiff.NewIndex.Primary))
- diffDetails = append(diffDetails, fmt.Sprintf(" New def: %s", indexDiff.NewIndex.Definition))
- } else if indexDiff.OldIndex != nil {
- oldExpr := ""
- if len(indexDiff.OldIndex.Expressions) > 0 {
- oldExpr = strings.Join(indexDiff.OldIndex.Expressions, ", ")
- }
- diffDetails = append(diffDetails, fmt.Sprintf(" Old: expr=%s type=%s unique=%v primary=%v", oldExpr, indexDiff.OldIndex.Type, indexDiff.OldIndex.Unique, indexDiff.OldIndex.Primary))
- diffDetails = append(diffDetails, fmt.Sprintf(" Old def: %s", indexDiff.OldIndex.Definition))
- } else if indexDiff.NewIndex != nil {
- newExpr := ""
- if len(indexDiff.NewIndex.Expressions) > 0 {
- newExpr = strings.Join(indexDiff.NewIndex.Expressions, ", ")
- }
- diffDetails = append(diffDetails, fmt.Sprintf(" New: expr=%s type=%s unique=%v primary=%v", newExpr, indexDiff.NewIndex.Type, indexDiff.NewIndex.Unique, indexDiff.NewIndex.Primary))
- diffDetails = append(diffDetails, fmt.Sprintf(" New def: %s", indexDiff.NewIndex.Definition))
- }
- }
- }
- }
- if len(diff.ViewChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("ViewChanges: %d", len(diff.ViewChanges)))
- for _, viewDiff := range diff.ViewChanges {
- diffDetails = append(diffDetails, fmt.Sprintf(" View: %s.%s, Action: %v", viewDiff.SchemaName, viewDiff.ViewName, viewDiff.Action))
- if viewDiff.OldView != nil && viewDiff.NewView != nil {
- diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))
- diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))
- } else if viewDiff.OldView != nil {
- diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))
- } else if viewDiff.NewView != nil {
- diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))
- }
- }
- }
- if len(diff.MaterializedViewChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("MaterializedViewChanges: %d", len(diff.MaterializedViewChanges)))
- }
- if len(diff.FunctionChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("FunctionChanges: %d", len(diff.FunctionChanges)))
- }
- if len(diff.ProcedureChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("ProcedureChanges: %d", len(diff.ProcedureChanges)))
- }
- if len(diff.SequenceChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("SequenceChanges: %d", len(diff.SequenceChanges)))
- }
- if len(diff.EnumTypeChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("EnumTypeChanges: %d", len(diff.EnumTypeChanges)))
- }
- if len(diff.EventChanges) > 0 {
- diffDetails = append(diffDetails, fmt.Sprintf("EventChanges: %d", len(diff.EventChanges)))
- }
-
- if len(diffDetails) > 0 {
- return errors.Errorf("schema D does not match expected schema C: found differences - %s", strings.Join(diffDetails, ", "))
- }
- }
-
- return nil
-}
-
-// DiffSummary represents the structure for JSON serialization of schema differences
-type DiffSummary struct {
- HasDifferences bool `json:"hasDifferences"`
- Summary string `json:"summary"`
- TableChanges int `json:"tableChanges"`
- ViewChanges int `json:"viewChanges"`
- FunctionChanges int `json:"functionChanges"`
- SchemaChanges int `json:"schemaChanges"`
- SequenceChanges int `json:"sequenceChanges"`
- EnumTypeChanges int `json:"enumTypeChanges"`
- Details *DiffDetails `json:"details,omitempty"`
-}
-
-// DiffDetails provides detailed information about the differences
-type DiffDetails struct {
- Tables []TableChangeSummary `json:"tables,omitempty"`
- Views []ViewChangeSummary `json:"views,omitempty"`
- Functions []FunctionChangeSummary `json:"functions,omitempty"`
- Schemas []SchemaChangeSummary `json:"schemas,omitempty"`
-}
-
-// TableChangeSummary summarizes table changes with detailed information
-type TableChangeSummary struct {
- Action string `json:"action"`
- SchemaName string `json:"schemaName"`
- TableName string `json:"tableName"`
- ColumnChanges int `json:"columnChanges"`
- IndexChanges int `json:"indexChanges"`
- ConstraintChanges int `json:"constraintChanges"`
- ForeignKeyChanges int `json:"foreignKeyChanges"`
- // Detailed changes
- Columns []ColumnChangeDetail `json:"columns,omitempty"`
- Indexes []IndexChangeDetail `json:"indexes,omitempty"`
- Constraints []ConstraintChangeDetail `json:"constraints,omitempty"`
- ForeignKeys []ForeignKeyChangeDetail `json:"foreignKeys,omitempty"`
-}
-
-// ViewChangeSummary summarizes view changes
-type ViewChangeSummary struct {
- Action string `json:"action"`
- SchemaName string `json:"schemaName"`
- ViewName string `json:"viewName"`
-}
-
-// FunctionChangeSummary summarizes function changes
-type FunctionChangeSummary struct {
- Action string `json:"action"`
- SchemaName string `json:"schemaName"`
- FunctionName string `json:"functionName"`
-}
-
-// SchemaChangeSummary summarizes schema changes
-type SchemaChangeSummary struct {
- Action string `json:"action"`
- SchemaName string `json:"schemaName"`
-}
-
-// ColumnChangeDetail provides detailed information about column changes
-type ColumnChangeDetail struct {
- Action string `json:"action"`
- Name string `json:"name"`
- OldColumn *ColumnInfo `json:"oldColumn,omitempty"`
- NewColumn *ColumnInfo `json:"newColumn,omitempty"`
-}
-
-// ColumnInfo represents column metadata
-type ColumnInfo struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Nullable bool `json:"nullable"`
- DefaultValue string `json:"defaultValue,omitempty"`
- Comment string `json:"comment,omitempty"`
-}
-
-// IndexChangeDetail provides detailed information about index changes
-type IndexChangeDetail struct {
- Action string `json:"action"`
- Name string `json:"name"`
- OldIndex *IndexInfo `json:"oldIndex,omitempty"`
- NewIndex *IndexInfo `json:"newIndex,omitempty"`
-}
-
-// IndexInfo represents index metadata
-type IndexInfo struct {
- Name string `json:"name"`
- Type string `json:"type,omitempty"`
- Unique bool `json:"unique"`
- Expressions []string `json:"expressions"`
- KeyLength []int64 `json:"keyLength,omitempty"`
- Descending []bool `json:"descending,omitempty"`
-}
-
-// ConstraintChangeDetail provides detailed information about constraint changes
-type ConstraintChangeDetail struct {
- Action string `json:"action"`
- Name string `json:"name"`
- OldConstraint *ConstraintInfo `json:"oldConstraint,omitempty"`
- NewConstraint *ConstraintInfo `json:"newConstraint,omitempty"`
-}
-
-// ConstraintInfo represents constraint metadata
-type ConstraintInfo struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Expression string `json:"expression,omitempty"`
-}
-
-// ForeignKeyChangeDetail provides detailed information about foreign key changes
-type ForeignKeyChangeDetail struct {
- Action string `json:"action"`
- Name string `json:"name"`
- OldForeignKey *ForeignKeyInfo `json:"oldForeignKey,omitempty"`
- NewForeignKey *ForeignKeyInfo `json:"newForeignKey,omitempty"`
-}
-
-// ForeignKeyInfo represents foreign key metadata
-type ForeignKeyInfo struct {
- Name string `json:"name"`
- Columns []string `json:"columns"`
- ReferencedSchema string `json:"referencedSchema"`
- ReferencedTable string `json:"referencedTable"`
- ReferencedColumns []string `json:"referencedColumns"`
- OnUpdate string `json:"onUpdate,omitempty"`
- OnDelete string `json:"onDelete,omitempty"`
- MatchType string `json:"matchType,omitempty"`
-}
-
-// createDiffSummary creates a structured summary from MetadataDiff
-func createDiffSummary(diff *schema.MetadataDiff) *DiffSummary {
- if diff == nil {
- return &DiffSummary{
- HasDifferences: false,
- Summary: "No schema differences found - schemas match perfectly!",
- }
- }
-
- // Check if there are actually any differences
- hasDifferences := len(diff.TableChanges) > 0 || len(diff.ViewChanges) > 0 ||
- len(diff.FunctionChanges) > 0 || len(diff.SchemaChanges) > 0 ||
- len(diff.SequenceChanges) > 0 || len(diff.EnumTypeChanges) > 0
-
- summary := &DiffSummary{
- HasDifferences: hasDifferences,
- TableChanges: len(diff.TableChanges),
- ViewChanges: len(diff.ViewChanges),
- FunctionChanges: len(diff.FunctionChanges),
- SchemaChanges: len(diff.SchemaChanges),
- SequenceChanges: len(diff.SequenceChanges),
- EnumTypeChanges: len(diff.EnumTypeChanges),
- }
-
- if !hasDifferences {
- summary.Summary = "No schema differences found - schemas match perfectly!"
- return summary
- }
-
- // Create summary text
- var summaryParts []string
- if summary.TableChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("Table changes: %d", summary.TableChanges))
- }
- if summary.ViewChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("View changes: %d", summary.ViewChanges))
- }
- if summary.FunctionChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("Function changes: %d", summary.FunctionChanges))
- }
- if summary.SchemaChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("Schema changes: %d", summary.SchemaChanges))
- }
- if summary.SequenceChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("Sequence changes: %d", summary.SequenceChanges))
- }
- if summary.EnumTypeChanges > 0 {
- summaryParts = append(summaryParts, fmt.Sprintf("Enum type changes: %d", summary.EnumTypeChanges))
- }
-
- summary.Summary = strings.Join(summaryParts, ", ")
-
- // Add detailed information
- details := &DiffDetails{}
-
- // Add table changes details
- for _, tableChange := range diff.TableChanges {
- tableSummary := TableChangeSummary{
- Action: string(tableChange.Action),
- SchemaName: tableChange.SchemaName,
- TableName: tableChange.TableName,
- ColumnChanges: len(tableChange.ColumnChanges),
- IndexChanges: len(tableChange.IndexChanges),
- ConstraintChanges: len(tableChange.CheckConstraintChanges),
- ForeignKeyChanges: len(tableChange.ForeignKeyChanges),
- }
-
- // Add detailed column changes
- for _, colChange := range tableChange.ColumnChanges {
- columnDetail := ColumnChangeDetail{
- Action: string(colChange.Action),
- Name: getColumnName(colChange),
- }
- if colChange.OldColumn != nil {
- columnDetail.OldColumn = &ColumnInfo{
- Name: colChange.OldColumn.Name,
- Type: colChange.OldColumn.Type,
- Nullable: colChange.OldColumn.Nullable,
- DefaultValue: colChange.OldColumn.Default,
- Comment: colChange.OldColumn.Comment,
- }
- }
- if colChange.NewColumn != nil {
- columnDetail.NewColumn = &ColumnInfo{
- Name: colChange.NewColumn.Name,
- Type: colChange.NewColumn.Type,
- Nullable: colChange.NewColumn.Nullable,
- DefaultValue: colChange.NewColumn.Default,
- Comment: colChange.NewColumn.Comment,
- }
- }
- tableSummary.Columns = append(tableSummary.Columns, columnDetail)
- }
-
- // Add detailed index changes
- for _, indexChange := range tableChange.IndexChanges {
- indexDetail := IndexChangeDetail{
- Action: string(indexChange.Action),
- Name: getIndexName(indexChange),
- }
- if indexChange.OldIndex != nil {
- indexDetail.OldIndex = &IndexInfo{
- Name: indexChange.OldIndex.Name,
- Type: indexChange.OldIndex.Type,
- Unique: indexChange.OldIndex.Unique,
- Expressions: indexChange.OldIndex.Expressions,
- KeyLength: indexChange.OldIndex.KeyLength,
- Descending: indexChange.OldIndex.Descending,
- }
- }
- if indexChange.NewIndex != nil {
- indexDetail.NewIndex = &IndexInfo{
- Name: indexChange.NewIndex.Name,
- Type: indexChange.NewIndex.Type,
- Unique: indexChange.NewIndex.Unique,
- Expressions: indexChange.NewIndex.Expressions,
- KeyLength: indexChange.NewIndex.KeyLength,
- Descending: indexChange.NewIndex.Descending,
- }
- }
- tableSummary.Indexes = append(tableSummary.Indexes, indexDetail)
- }
-
- // Add detailed constraint changes
- for _, constraintChange := range tableChange.CheckConstraintChanges {
- constraintDetail := ConstraintChangeDetail{
- Action: string(constraintChange.Action),
- Name: getConstraintName(constraintChange),
- }
- if constraintChange.OldCheckConstraint != nil {
- constraintDetail.OldConstraint = &ConstraintInfo{
- Name: constraintChange.OldCheckConstraint.Name,
- Type: "CHECK",
- Expression: constraintChange.OldCheckConstraint.Expression,
- }
- }
- if constraintChange.NewCheckConstraint != nil {
- constraintDetail.NewConstraint = &ConstraintInfo{
- Name: constraintChange.NewCheckConstraint.Name,
- Type: "CHECK",
- Expression: constraintChange.NewCheckConstraint.Expression,
- }
- }
- tableSummary.Constraints = append(tableSummary.Constraints, constraintDetail)
- }
-
- // Add detailed foreign key changes
- for _, fkChange := range tableChange.ForeignKeyChanges {
- fkDetail := ForeignKeyChangeDetail{
- Action: string(fkChange.Action),
- Name: getForeignKeyName(fkChange),
- }
- if fkChange.OldForeignKey != nil {
- fkDetail.OldForeignKey = &ForeignKeyInfo{
- Name: fkChange.OldForeignKey.Name,
- Columns: fkChange.OldForeignKey.Columns,
- ReferencedSchema: fkChange.OldForeignKey.ReferencedSchema,
- ReferencedTable: fkChange.OldForeignKey.ReferencedTable,
- ReferencedColumns: fkChange.OldForeignKey.ReferencedColumns,
- OnUpdate: fkChange.OldForeignKey.OnUpdate,
- OnDelete: fkChange.OldForeignKey.OnDelete,
- MatchType: fkChange.OldForeignKey.MatchType,
- }
- }
- if fkChange.NewForeignKey != nil {
- fkDetail.NewForeignKey = &ForeignKeyInfo{
- Name: fkChange.NewForeignKey.Name,
- Columns: fkChange.NewForeignKey.Columns,
- ReferencedSchema: fkChange.NewForeignKey.ReferencedSchema,
- ReferencedTable: fkChange.NewForeignKey.ReferencedTable,
- ReferencedColumns: fkChange.NewForeignKey.ReferencedColumns,
- OnUpdate: fkChange.NewForeignKey.OnUpdate,
- OnDelete: fkChange.NewForeignKey.OnDelete,
- MatchType: fkChange.NewForeignKey.MatchType,
- }
- }
- tableSummary.ForeignKeys = append(tableSummary.ForeignKeys, fkDetail)
- }
-
- details.Tables = append(details.Tables, tableSummary)
- }
-
- // Add view changes details
- for _, viewChange := range diff.ViewChanges {
- details.Views = append(details.Views, ViewChangeSummary{
- Action: string(viewChange.Action),
- SchemaName: viewChange.SchemaName,
- ViewName: viewChange.ViewName,
- })
- }
-
- // Add function changes details
- for _, funcChange := range diff.FunctionChanges {
- details.Functions = append(details.Functions, FunctionChangeSummary{
- Action: string(funcChange.Action),
- SchemaName: funcChange.SchemaName,
- FunctionName: funcChange.FunctionName,
- })
- }
-
- // Add schema changes details
- for _, schemaChange := range diff.SchemaChanges {
- details.Schemas = append(details.Schemas, SchemaChangeSummary{
- Action: string(schemaChange.Action),
- SchemaName: schemaChange.SchemaName,
- })
- }
-
- if len(details.Tables) > 0 || len(details.Views) > 0 || len(details.Functions) > 0 || len(details.Schemas) > 0 {
- summary.Details = details
- }
-
- return summary
-}
-
-// Helper functions to extract names from diff structures
-func getColumnName(colChange *schema.ColumnDiff) string {
- if colChange.NewColumn != nil {
- return colChange.NewColumn.Name
- }
- if colChange.OldColumn != nil {
- return colChange.OldColumn.Name
- }
- return ""
-}
-
-func getIndexName(indexChange *schema.IndexDiff) string {
- if indexChange.NewIndex != nil {
- return indexChange.NewIndex.Name
- }
- if indexChange.OldIndex != nil {
- return indexChange.OldIndex.Name
- }
- return ""
-}
-
-func getConstraintName(constraintChange *schema.CheckConstraintDiff) string {
- if constraintChange.NewCheckConstraint != nil {
- return constraintChange.NewCheckConstraint.Name
- }
- if constraintChange.OldCheckConstraint != nil {
- return constraintChange.OldCheckConstraint.Name
- }
- return ""
-}
-
-func getForeignKeyName(fkChange *schema.ForeignKeyDiff) string {
- if fkChange.NewForeignKey != nil {
- return fkChange.NewForeignKey.Name
- }
- if fkChange.OldForeignKey != nil {
- return fkChange.OldForeignKey.Name
- }
- return ""
-}
-
-// validateOrSaveTestFiles validates existing test files or creates new ones based on UPDATE_TEST_FILES environment variable
-func validateOrSaveTestFiles(t *testing.T, diff *schema.MetadataDiff, testName string, ddlContent string) error {
- // Check if we should update test files
- updateFiles := os.Getenv("UPDATE_TEST_FILES") == "true"
-
- testDir := getTestDataDirectory(testName)
- if testDir == "" {
- // For hardcoded tests (TestSDLValidationWithTestcontainer), skip file validation
- // Only TestSDLValidationFromTestData tests should have corresponding test data files
- if strings.Contains(testName, "TestSDLValidationWithTestcontainer") {
- t.Logf("Skipping file validation for hardcoded test: %s", testName)
- return nil
- }
- return errors.Errorf("could not determine test data directory for test %s", testName)
- }
-
- diffFilename := filepath.Join(testDir, "diff.json")
- ddlFilename := filepath.Join(testDir, "ddl.sql")
-
- // Generate current content
- currentDiffJSON, err := generateDiffJSON(diff)
- if err != nil {
- return errors.Wrapf(err, "failed to generate diff JSON")
- }
-
- if updateFiles {
- // Update mode: create/overwrite files
- t.Logf("Updating test files in %s", testDir)
-
- // Save diff JSON file
- if err := os.WriteFile(diffFilename, []byte(currentDiffJSON), 0644); err != nil {
- return errors.Wrapf(err, "failed to write diff.json file")
- }
-
- // Save DDL file
- if err := os.WriteFile(ddlFilename, []byte(ddlContent), 0644); err != nil {
- return errors.Wrapf(err, "failed to write ddl.sql file")
- }
-
- t.Logf("Test files updated: %s (diff.json, ddl.sql)", testDir)
- return nil
- }
-
- // Validation mode: check if existing files match current content
- t.Logf("Validating test files in %s", testDir)
-
- // Validate diff.json
- if err := validateFileContent(diffFilename, currentDiffJSON, "diff.json"); err != nil {
- return err
- }
-
- // Validate ddl.sql
- if err := validateFileContent(ddlFilename, ddlContent, "ddl.sql"); err != nil {
- return err
- }
-
- t.Logf("✓ Test files validated successfully: %s", testDir)
- return nil
-}
-
-// generateDiffJSON generates JSON content from MetadataDiff
-func generateDiffJSON(diff *schema.MetadataDiff) (string, error) {
- diffSummary := createDiffSummary(diff)
- jsonData, err := json.MarshalIndent(diffSummary, "", " ")
- if err != nil {
- return "", errors.Wrapf(err, "failed to marshal diff to JSON")
- }
- return string(jsonData), nil
-}
-
-// validateFileContent validates that the file content matches expected content
-func validateFileContent(filename, expectedContent, fileType string) error {
- if _, err := os.Stat(filename); os.IsNotExist(err) {
- return errors.Errorf("%s file does not exist: %s. Run with UPDATE_TEST_FILES=true to create it", fileType, filename)
- }
-
- actualContent, err := os.ReadFile(filename)
- if err != nil {
- return errors.Wrapf(err, "failed to read %s file: %s", fileType, filename)
- }
-
- actualStr := strings.TrimSpace(string(actualContent))
- expectedStr := strings.TrimSpace(expectedContent)
-
- if actualStr != expectedStr {
- return errors.Errorf("%s content mismatch in %s.\nExpected:\n%s\n\nActual:\n%s\n\nRun with UPDATE_TEST_FILES=true to update the file",
- fileType, filename, expectedStr, actualStr)
- }
-
- return nil
-}
-
-// getTestDataDirectory converts a test name to the corresponding test data directory path
-func getTestDataDirectory(testName string) string {
- // Handle TestSDLValidationFromTestData_ prefix
- var pathPart string
- if strings.HasPrefix(testName, "TestSDLValidationFromTestData_") {
- pathPart = strings.TrimPrefix(testName, "TestSDLValidationFromTestData_")
- } else {
- return ""
- }
-
- // Convert from test name format to directory format
- // TestSDLValidationFromTestData_data_types_basic_types_datetime_types -> data_types/basic_types/datetime_types
-
- // Based on the known test structure, parse the path correctly:
- // data_types_basic_types_datetime_types
- // schema_objects_views_complex_views
- // constraints_indexes_foreign_keys_cascade_actions
-
- var testDataPath string
- switch {
- case strings.HasPrefix(pathPart, "data_types_basic_types_"):
- // data_types_basic_types_datetime_types -> data_types/basic_types/datetime_types
- suffix := strings.TrimPrefix(pathPart, "data_types_basic_types_")
- testDataPath = filepath.Join("data_types", "basic_types", suffix)
- case strings.HasPrefix(pathPart, "data_types_advanced_types_"):
- // data_types_advanced_types_binary_json_uuid -> data_types/advanced_types/binary_json_uuid
- suffix := strings.TrimPrefix(pathPart, "data_types_advanced_types_")
- testDataPath = filepath.Join("data_types", "advanced_types", suffix)
- case strings.HasPrefix(pathPart, "data_types_type_conversions_"):
- // data_types_type_conversions_compatible_conversions -> data_types/type_conversions/compatible_conversions
- suffix := strings.TrimPrefix(pathPart, "data_types_type_conversions_")
- testDataPath = filepath.Join("data_types", "type_conversions", suffix)
- case strings.HasPrefix(pathPart, "schema_objects_views_"):
- // schema_objects_views_complex_views -> schema_objects/views/complex_views
- suffix := strings.TrimPrefix(pathPart, "schema_objects_views_")
- testDataPath = filepath.Join("schema_objects", "views", suffix)
- case strings.HasPrefix(pathPart, "constraints_indexes_foreign_keys_"):
- // constraints_indexes_foreign_keys_cascade_actions -> constraints_indexes/foreign_keys/cascade_actions
- suffix := strings.TrimPrefix(pathPart, "constraints_indexes_foreign_keys_")
- testDataPath = filepath.Join("constraints_indexes", "foreign_keys", suffix)
- case strings.HasPrefix(pathPart, "table_operations_column_operations_"):
- // table_operations_column_operations_add_simple_columns -> table_operations/column_operations/add_simple_columns
- suffix := strings.TrimPrefix(pathPart, "table_operations_column_operations_")
- testDataPath = filepath.Join("table_operations", "column_operations", suffix)
- default:
- // Fallback: try to split intelligently
- parts := strings.Split(pathPart, "_")
- if len(parts) >= 3 {
- // Take first 2 parts as category, rest as test name
- testDataPath = filepath.Join(strings.Join(parts[:2], "/"), strings.Join(parts[2:], "_"))
- } else {
- testDataPath = strings.ReplaceAll(pathPart, "_", "/")
- }
- }
-
- return filepath.Join("sdl_testdata", testDataPath)
-}
diff --git a/backend/plugin/schema/pg/walk_through_omni.go b/backend/plugin/schema/pg/walk_through_omni.go
index 8a3d46d5f9..fa864dbb5a 100644
--- a/backend/plugin/schema/pg/walk_through_omni.go
+++ b/backend/plugin/schema/pg/walk_through_omni.go
@@ -1,14 +1,17 @@
package pg
import (
+ "context"
"fmt"
"slices"
+ "strings"
"github.com/bytebase/omni/pg/catalog"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
"github.com/bytebase/bytebase/backend/plugin/schema"
"github.com/bytebase/bytebase/backend/store/model"
)
@@ -466,3 +469,55 @@ func fkMatchToString(match byte) string {
return "SIMPLE"
}
}
+
+func extractViewDependencies(schemaMetadata *storepb.DatabaseSchemaMetadata) {
+ for _, s := range schemaMetadata.Schemas {
+ for _, view := range s.Views {
+ view.DependencyColumns = getViewDependencies(view.Definition, s.Name, schemaMetadata)
+ }
+ for _, mv := range s.MaterializedViews {
+ mv.DependencyColumns = getViewDependencies(mv.Definition, s.Name, schemaMetadata)
+ }
+ }
+}
+
+func getViewDependencies(viewDef string, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {
+ queryStatement := strings.TrimSpace(viewDef)
+ span, err := pgparser.GetQuerySpan(
+ context.Background(),
+ base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {
+ dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, nil, nil, storepb.Engine_POSTGRES, false)
+ return databaseName, dbMetadata, nil
+ },
+ ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {
+ return []string{}, nil
+ },
+ },
+ base.Statement{Text: queryStatement},
+ "",
+ schemaName,
+ false,
+ )
+ if err != nil {
+ return []*storepb.DependencyColumn{}
+ }
+
+ dependencyMap := make(map[string]*storepb.DependencyColumn)
+ for sourceColumn := range span.SourceColumns {
+ key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)
+ if _, exists := dependencyMap[key]; !exists {
+ dependencyMap[key] = &storepb.DependencyColumn{
+ Schema: sourceColumn.Schema,
+ Table: sourceColumn.Table,
+ Column: "*",
+ }
+ }
+ }
+
+ var dependencies []*storepb.DependencyColumn
+ for _, dep := range dependencyMap {
+ dependencies = append(dependencies, dep)
+ }
+ return dependencies
+}
diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.go
index 74f711da98..5bf9a030a0 100644
--- a/backend/plugin/schema/schema.go
+++ b/backend/plugin/schema/schema.go
@@ -24,9 +24,8 @@ var (
getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)
generateMigrations = make(map[storepb.Engine]generateMigration)
getSDLDiffs = make(map[storepb.Engine]getSDLDiff)
- sdlMigrations = make(map[storepb.Engine]sdlMigration)
sdlDropAdvicesFns = make(map[storepb.Engine]sdlDropAdvices)
- schemaDiffMigrations = make(map[storepb.Engine]schemaDiffMigration)
+ diffSDLMigrations = make(map[storepb.Engine]diffSDLMigration)
getMultiFileDatabaseDefinitions = make(map[storepb.Engine]getMultiFileDatabaseDefinition)
walkThroughs = make(map[storepb.Engine]walkThrough)
walkThroughsWithContext = make(map[storepb.Engine]walkThroughWithContext)
@@ -44,9 +43,8 @@ type getSequenceDefinition func(string, *storepb.SequenceMetadata) (string, erro
type getDatabaseMetadata func(string) (*storepb.DatabaseSchemaMetadata, error)
type generateMigration func(*MetadataDiff) (string, error)
type getSDLDiff func(currentSDLText, previousUserSDLText string, currentSchema *model.DatabaseMetadata) (*MetadataDiff, error)
-type sdlMigration func(userSDLText string, currentSchema *model.DatabaseMetadata) (string, error)
type sdlDropAdvices func(userSDLText string, currentSchema *model.DatabaseMetadata) ([]*storepb.Advice, error)
-type schemaDiffMigration func(oldSchema, newSchema *model.DatabaseMetadata) (string, error)
+type diffSDLMigration func(sourceSDL, targetSDL string) (string, error)
type walkThrough func(*model.DatabaseMetadata, []base.AST) *storepb.Advice
type walkThroughWithContext func(WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice
@@ -268,23 +266,15 @@ func GetSDLDiff(engine storepb.Engine, currentSDLText, previousUserSDLText strin
return f(currentSDLText, previousUserSDLText, currentSchema)
}
-func RegisterSDLMigration(engine storepb.Engine, f sdlMigration) {
- mux.Lock()
- defer mux.Unlock()
- if _, dup := sdlMigrations[engine]; dup {
- panic(fmt.Sprintf("Register called twice %s", engine))
- }
- sdlMigrations[engine] = f
-}
-
// SDLMigration computes the migration SQL from a user-provided SDL text and the
-// current database schema. It combines diff and migration generation in one step.
+// current database schema. It converts the current metadata to SDL, then diffs
+// against the user SDL using DiffSDLMigration.
func SDLMigration(engine storepb.Engine, userSDLText string, currentSchema *model.DatabaseMetadata) (string, error) {
- f, ok := sdlMigrations[engine]
- if !ok {
- return "", errors.Errorf("engine %s is not supported for SDL migration", engine)
+ sourceSDL, err := MetadataToSDL(engine, currentSchema)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to convert current schema to SDL")
}
- return f(userSDLText, currentSchema)
+ return DiffSDLMigration(engine, sourceSDL, userSDLText)
}
func RegisterSDLDropAdvices(engine storepb.Engine, f sdlDropAdvices) {
@@ -305,21 +295,20 @@ func SDLDropAdvices(engine storepb.Engine, userSDLText string, currentSchema *mo
return f(userSDLText, currentSchema)
}
-func RegisterDiffMigration(engine storepb.Engine, f schemaDiffMigration) {
- mux.Lock()
- defer mux.Unlock()
- if _, dup := schemaDiffMigrations[engine]; dup {
- panic(fmt.Sprintf("Register called twice %s", engine))
- }
- schemaDiffMigrations[engine] = f
-}
-
// DiffMigration computes the migration SQL between two database metadata states.
-// Falls back to the legacy GetDatabaseSchemaDiff + GenerateMigration path if no
-// engine-specific implementation is registered.
+// For engines with DiffSDLMigration registered, converts both sides to SDL and diffs.
+// Falls back to the legacy GetDatabaseSchemaDiff + GenerateMigration path otherwise.
func DiffMigration(engine storepb.Engine, oldSchema, newSchema *model.DatabaseMetadata) (string, error) {
- if f, ok := schemaDiffMigrations[engine]; ok {
- return f(oldSchema, newSchema)
+ if _, ok := diffSDLMigrations[engine]; ok {
+ sourceSDL, err := MetadataToSDL(engine, oldSchema)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to convert source schema to SDL")
+ }
+ targetSDL, err := MetadataToSDL(engine, newSchema)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to convert target schema to SDL")
+ }
+ return DiffSDLMigration(engine, sourceSDL, targetSDL)
}
// Fallback to legacy path for engines that haven't migrated yet.
diff, err := GetDatabaseSchemaDiff(engine, oldSchema, newSchema)
@@ -329,6 +318,39 @@ func DiffMigration(engine storepb.Engine, oldSchema, newSchema *model.DatabaseMe
return GenerateMigration(engine, diff)
}
+// MetadataToSDL converts database metadata to SDL text using the registered GetDatabaseDefinition.
+func MetadataToSDL(engine storepb.Engine, meta *model.DatabaseMetadata) (string, error) {
+ if meta == nil {
+ return "", nil
+ }
+ proto := meta.GetProto()
+ if proto == nil {
+ return "", nil
+ }
+ return GetDatabaseDefinition(engine, GetDefinitionContext{
+ SkipBackupSchema: true,
+ SDLFormat: true,
+ }, proto)
+}
+
+func RegisterDiffSDLMigration(engine storepb.Engine, f diffSDLMigration) {
+ mux.Lock()
+ defer mux.Unlock()
+ if _, dup := diffSDLMigrations[engine]; dup {
+ panic(fmt.Sprintf("Register called twice %s", engine))
+ }
+ diffSDLMigrations[engine] = f
+}
+
+// DiffSDLMigration computes migration SQL between two SDL texts.
+func DiffSDLMigration(engine storepb.Engine, sourceSDL, targetSDL string) (string, error) {
+ f, ok := diffSDLMigrations[engine]
+ if !ok {
+ return "", errors.Errorf("engine %s is not supported for SDL diff migration", engine)
+ }
+ return f(sourceSDL, targetSDL)
+}
+
func RegisterGetMultiFileDatabaseDefinition(engine storepb.Engine, f getMultiFileDatabaseDefinition) {
mux.Lock()
defer mux.Unlock()