Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: use extract access table to get dependency (#17924)
rebeliceb16813a0ee36
diff --git a/backend/plugin/parser/pg/access_tables.go b/backend/plugin/parser/pg/access_tables.goindex ae508c1445..a02377cf44 100644--- a/backend/plugin/parser/pg/access_tables.go+++ b/backend/plugin/parser/pg/access_tables.go@@ -3,21 +3,71 @@ package pgimport ("context"+ "github.com/antlr4-go/antlr/v4""github.com/bytebase/parser/postgresql""github.com/pkg/errors""github.com/bytebase/bytebase/backend/plugin/parser/base")+// ExtractAccessTablesOption configures access table extraction behavior.+type ExtractAccessTablesOption struct {+ // DefaultDatabase is the default database name to use when table reference doesn't specify one.+ DefaultDatabase string+ // DefaultSchema is the default schema name to use when table reference doesn't specify one.+ DefaultSchema string+ // SkipMetadataValidation skips metadata lookup and validation.+ // When true, returns all table references found in SQL without checking if they exist.+ SkipMetadataValidation bool+ // GetDatabaseMetadata is the function to get database metadata (optional, only used when SkipMetadataValidation is false).+ GetDatabaseMetadata base.GetDatabaseMetadataFunc+ // Ctx is the context for metadata lookup (optional, only used when SkipMetadataValidation is false).+ Ctx context.Context+ // InstanceID is the instance ID for metadata lookup (optional, only used when SkipMetadataValidation is false).+ InstanceID string+}++// ExtractAccessTables extracts all table/view references from a SQL statement.+// This is a lightweight version that doesn't perform full query span analysis.+func ExtractAccessTables(statement string, option ExtractAccessTablesOption) ([]base.ColumnResource, error) {+ parseResult, err := ParsePostgreSQL(statement)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to parse PostgreSQL statement")+ }++ searchPath := []string{option.DefaultSchema}+ if option.DefaultSchema == "" {+ searchPath = []string{"public"}+ }++ extractor := &accessTableExtractor{+ defaultDatabase: option.DefaultDatabase,+ searchPath: searchPath,+ getDatabaseMetadata: option.GetDatabaseMetadata,+ ctx: option.Ctx,+ instanceID: option.InstanceID,+ skipMetadataValidation: option.SkipMetadataValidation,+ }++ antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)++ if extractor.err != nil {+ return nil, errors.Wrapf(extractor.err, "failed to extract access tables")+ }++ return extractor.accessTables, nil+}+type accessTableExtractor struct {*postgresql.BasePostgreSQLParserListener- err error- defaultDatabase string- searchPath []string- accessTables []base.ColumnResource- getDatabaseMetadata base.GetDatabaseMetadataFunc- ctx context.Context- instanceID string+ err error+ defaultDatabase string+ searchPath []string+ accessTables []base.ColumnResource+ getDatabaseMetadata base.GetDatabaseMetadataFunc+ ctx context.Context+ instanceID string+ skipMetadataValidation bool}func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_nameContext) {@@ -57,6 +107,17 @@ func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_namereturn}+ if a.skipMetadataValidation {+ // If schema is not specified, use the first schema in search path as default+ if resource.Schema == "" && !isSystemResource(resource) {+ if len(a.searchPath) > 0 {+ resource.Schema = a.searchPath[0]+ }+ }+ a.accessTables = append(a.accessTables, resource)+ return+ }+if !isSystemResource(resource) {searchPath := a.searchPathif resource.Schema != "" {@@ -66,6 +127,7 @@ func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_name_, databaseMetadata, err := a.getDatabaseMetadata(a.ctx, a.instanceID, a.defaultDatabase)if err != nil {a.err = errors.Wrapf(err, "failed to get database metadata for database: %s", a.defaultDatabase)+ return}// Access pseudo table or table/view we do not sync, return directly.if databaseMetadata == nil {diff --git a/backend/plugin/schema/pg/generate_migration.go b/backend/plugin/schema/pg/generate_migration.goindex bbb87bf85d..dfe7134bb2 100644--- a/backend/plugin/schema/pg/generate_migration.go+++ b/backend/plugin/schema/pg/generate_migration.go@@ -1,7 +1,6 @@package pgimport (- "context""fmt""slices""strconv"@@ -16,7 +15,6 @@ import (pgpluginparser "github.com/bytebase/bytebase/backend/plugin/parser/pg""github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"- "github.com/bytebase/bytebase/backend/store/model")func init() {@@ -718,8 +716,10 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) errorfor _, dep := range dependencies {depID := getMigrationObjectID(dep.Schema, dep.Table)- // Edge from dependency to dependent (table/view to view)- graph.AddEdge(depID, viewID)+ if allObjects[depID] {+ // Edge from dependency to dependent (table/view to view)+ graph.AddEdge(depID, viewID)+ }}}@@ -3427,18 +3427,17 @@ func (v *CreateOrReplaceVisitor) hasOrReplace(ctx *pgparser.CreatefunctionstmtCoreturn false}-// getViewDependenciesFromAST extracts view dependencies from AST node-// This reuses the same logic as getViewDependencies from get_database_metadata.go-func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {+// getViewDependenciesFromAST extracts view dependencies from AST node using lightweight access table extraction.+// This avoids the complexity and potential panics of GetQuerySpan by only extracting table/view references.+// The caller is responsible for filtering dependencies against the set of objects being migrated.+func getViewDependenciesFromAST(astNode any, schemaName string, _ *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {if astNode == nil {return []*storepb.DependencyColumn{}}- // Extract the SELECT statement from the VIEW AST nodevar selectStatement stringif ctx, ok := astNode.(*pgparser.ViewstmtContext); ok {- // Extract the SELECT statement from the ViewstmtContextif ctx.Selectstmt() != nil {// Try to get text using token stream firstif tokenStream := ctx.GetParser().GetTokenStream(); tokenStream != nil {@@ -3449,7 +3448,7 @@ func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetada}}- // Fallback to token-based approach if token stream approach failed+ // Fallback to token-based approach if failedif selectStatement == "" {selectStatement = getTextFromAST(ctx.Selectstmt())}@@ -3460,60 +3459,39 @@ func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetadareturn []*storepb.DependencyColumn{}}- // Use the same dependency extraction logic as getViewDependenciesqueryStatement := strings.TrimSpace(selectStatement)- // Use GetQuerySpan with the full schema metadata so it can resolve table/view references- // Wrap in defer/recover to handle potential panics from incomplete metadata- var span *base.QuerySpan- var err error- func() {- defer func() {- if r := recover(); r != nil {- // If GetQuerySpan panics (e.g., due to incomplete metadata), treat it as an error- err = errors.Errorf("GetQuerySpan panicked: %v", r)- }- }()- span, err = pgpluginparser.GetQuerySpan(- context.Background(),- base.GetQuerySpanContext{- GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {- // Return the full schema metadata so GetQuerySpan can resolve references- dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, false, false)- return databaseName, dbMetadata, nil- },- ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {- // Return empty list - we don't need actual database names for dependency extraction- return []string{}, nil- },- },- queryStatement,- "", // database- schemaName,- false, // case sensitive- )- }()-- // If error parsing query span, return empty dependencies+ accessTables, err := pgpluginparser.ExtractAccessTables(queryStatement, pgpluginparser.ExtractAccessTablesOption{+ DefaultDatabase: "",+ DefaultSchema: schemaName,+ SkipMetadataValidation: true,+ })if err != nil {- return []*storepb.DependencyColumn{} // nolint:nilerr+ return []*storepb.DependencyColumn{}}- // Collect unique dependencies+ // The caller will filter against allObjectsdependencyMap := make(map[string]*storepb.DependencyColumn)- for sourceColumn := range span.SourceColumns {- // Create dependency key in format: schema.table- key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)+ for _, resource := range accessTables {+ if resource.Schema == "pg_catalog" || resource.Schema == "information_schema" {+ continue+ }++ resourceSchema := resource.Schema+ if resourceSchema == "" {+ resourceSchema = schemaName+ }++ key := fmt.Sprintf("%s.%s", resourceSchema, resource.Table)if _, exists := dependencyMap[key]; !exists {dependencyMap[key] = &storepb.DependencyColumn{- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- Column: "*", // Use wildcard since we're tracking table-level dependencies+ Schema: resourceSchema,+ Table: resource.Table,+ Column: "*", // Table-level dependencies}}}- // Convert map to slicevar dependencies []*storepb.DependencyColumnfor _, dep := range dependencyMap {dependencies = append(dependencies, dep)diff --git a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go b/backend/plugin/schema/pg/generate_migration_testcontainer_test.goindex 65dd7ff1c1..673a302552 100644--- a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go+++ b/backend/plugin/schema/pg/generate_migration_testcontainer_test.go@@ -2511,3 +2511,97 @@ CREATE TABLE table2 (require.NoError(t, err)require.True(t, fkExists, "FK constraint fk_table1_name should exist")}++// TestSDLViewDependencyChain tests that SDL migration correctly handles view dependencies+// when a table is depended on by view1, and view1 is depended on by view2+func TestSDLViewDependencyChain(t *testing.T) {+ ctx := context.Background()++ // Start PostgreSQL container+ pgContainer, err := postgres.Run(ctx,+ "postgres:16-alpine",+ postgres.WithDatabase("test_db"),+ postgres.WithUsername("postgres"),+ postgres.WithPassword("test"),+ testcontainers.WithWaitStrategy(+ wait.ForLog("database system is ready to accept connections").+ WithOccurrence(2).+ WithStartupTimeout(5*time.Minute),+ ),+ )+ require.NoError(t, err)+ defer func() {+ if err := pgContainer.Terminate(ctx); err != nil {+ t.Logf("failed to terminate container: %s", err)+ }+ }()++ // Get connection string+ connectionString, err := pgContainer.ConnectionString(ctx, "sslmode=disable")+ require.NoError(t, err)++ // Connect to the database+ connConfig, err := pgx.ParseConfig(connectionString)+ require.NoError(t, err)+ testDB := stdlib.OpenDB(*connConfig)+ defer testDB.Close()++ // This SDL text simulates the REAL multi-file scenario where files are merged alphabetically:+ // Files: table_base.sql, test_view_0.sql, test_view2.sql+ // Alphabetical order: table_base < test_view_0 < test_view2+ // Dependency chain: table_base -> test_view2 -> test_view_0+ // So SDL has: table_base, test_view_0 (depends on test_view2 which comes AFTER), test_view2+ currentSDL := `+CREATE TABLE table_base (+ id serial,+ info text not null,+ created_at timestamp default current_timestamp,+ updated_at timestamp default current_timestamp,+ constraint pk_table_base_id primary key (id)+);++CREATE VIEW test_view_0 AS+SELECT * FROM test_view2;++CREATE VIEW test_view2 AS+SELECT id, info, created_at, updated_at FROM table_base;+`++ previousSDL := ``++ // Get current schema (empty database)+ currentSchema, err := getSyncMetadataForGenerateMigration(ctx, connConfig, connConfig.Database)+ require.NoError(t, err)+ dbSchema := model.NewDatabaseSchema(currentSchema, nil, nil, storepb.Engine_POSTGRES, false)++ // Get SDL diff+ diff, err := GetSDLDiff(currentSDL, previousSDL, dbSchema, nil)+ require.NoError(t, err)++ // Generate migration SQL with topological sorting+ migrationSQL, err := generateMigration(diff)+ require.NoError(t, err)++ // Execute the generated migration SQL+ // This should work because generateMigration should have sorted objects by dependencies+ _, err = testDB.Exec(migrationSQL)+ require.NoError(t, err, "Migration SQL should execute successfully with correct view dependency order")++ // Verify table was created+ var tableExists bool+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'table_base')").Scan(&tableExists)+ require.NoError(t, err)+ require.True(t, tableExists, "table_base should be created")++ // Verify test_view2 was created (depends on table_base)+ var view2Exists bool+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.views WHERE table_name = 'test_view2')").Scan(&view2Exists)+ require.NoError(t, err)+ require.True(t, view2Exists, "test_view2 should be created")++ // Verify test_view_0 was created (depends on test_view2)+ var view0Exists bool+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.views WHERE table_name = 'test_view_0')").Scan(&view0Exists)+ require.NoError(t, err)+ require.True(t, view0Exists, "test_view_0 should be created")+}diff --git a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.goindex 4641b28bf6..04a40c551c 100644--- a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go+++ b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go@@ -1156,6 +1156,17 @@ func validateSchemaConsistency(schemaD, schemaC *storepb.DatabaseSchemaMetadata)}if len(diff.ViewChanges) > 0 {diffDetails = append(diffDetails, fmt.Sprintf("ViewChanges: %d", len(diff.ViewChanges)))+ for _, viewDiff := range diff.ViewChanges {+ diffDetails = append(diffDetails, fmt.Sprintf(" View: %s.%s, Action: %v", viewDiff.SchemaName, viewDiff.ViewName, viewDiff.Action))+ if viewDiff.OldView != nil && viewDiff.NewView != nil {+ diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))+ diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))+ } else if viewDiff.OldView != nil {+ diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))+ } else if viewDiff.NewView != nil {+ diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))+ }+ }}if len(diff.MaterializedViewChanges) > 0 {diffDetails = append(diffDetails, fmt.Sprintf("MaterializedViewChanges: %d", len(diff.MaterializedViewChanges)))