Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore: use extract access table to get dependency (#17924)

rebeliceb16813a0ee36
diff --git a/backend/plugin/parser/pg/access_tables.go b/backend/plugin/parser/pg/access_tables.go
index ae508c1445..a02377cf44 100644
--- a/backend/plugin/parser/pg/access_tables.go
+++ b/backend/plugin/parser/pg/access_tables.go
@@ -3,21 +3,71 @@ package pg
import (
"context"
+ "github.com/antlr4-go/antlr/v4"
"github.com/bytebase/parser/postgresql"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
+// ExtractAccessTablesOption configures access table extraction behavior.
+type ExtractAccessTablesOption struct {
+ // DefaultDatabase is the default database name to use when table reference doesn't specify one.
+ DefaultDatabase string
+ // DefaultSchema is the default schema name to use when table reference doesn't specify one.
+ DefaultSchema string
+ // SkipMetadataValidation skips metadata lookup and validation.
+ // When true, returns all table references found in SQL without checking if they exist.
+ SkipMetadataValidation bool
+ // GetDatabaseMetadata is the function to get database metadata (optional, only used when SkipMetadataValidation is false).
+ GetDatabaseMetadata base.GetDatabaseMetadataFunc
+ // Ctx is the context for metadata lookup (optional, only used when SkipMetadataValidation is false).
+ Ctx context.Context
+ // InstanceID is the instance ID for metadata lookup (optional, only used when SkipMetadataValidation is false).
+ InstanceID string
+}
+
+// ExtractAccessTables extracts all table/view references from a SQL statement.
+// This is a lightweight version that doesn't perform full query span analysis.
+func ExtractAccessTables(statement string, option ExtractAccessTablesOption) ([]base.ColumnResource, error) {
+ parseResult, err := ParsePostgreSQL(statement)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to parse PostgreSQL statement")
+ }
+
+ searchPath := []string{option.DefaultSchema}
+ if option.DefaultSchema == "" {
+ searchPath = []string{"public"}
+ }
+
+ extractor := &accessTableExtractor{
+ defaultDatabase: option.DefaultDatabase,
+ searchPath: searchPath,
+ getDatabaseMetadata: option.GetDatabaseMetadata,
+ ctx: option.Ctx,
+ instanceID: option.InstanceID,
+ skipMetadataValidation: option.SkipMetadataValidation,
+ }
+
+ antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)
+
+ if extractor.err != nil {
+ return nil, errors.Wrapf(extractor.err, "failed to extract access tables")
+ }
+
+ return extractor.accessTables, nil
+}
+
type accessTableExtractor struct {
*postgresql.BasePostgreSQLParserListener
- err error
- defaultDatabase string
- searchPath []string
- accessTables []base.ColumnResource
- getDatabaseMetadata base.GetDatabaseMetadataFunc
- ctx context.Context
- instanceID string
+ err error
+ defaultDatabase string
+ searchPath []string
+ accessTables []base.ColumnResource
+ getDatabaseMetadata base.GetDatabaseMetadataFunc
+ ctx context.Context
+ instanceID string
+ skipMetadataValidation bool
}
func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_nameContext) {
@@ -57,6 +107,17 @@ func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_name
return
}
+ if a.skipMetadataValidation {
+ // If schema is not specified, use the first schema in search path as default
+ if resource.Schema == "" && !isSystemResource(resource) {
+ if len(a.searchPath) > 0 {
+ resource.Schema = a.searchPath[0]
+ }
+ }
+ a.accessTables = append(a.accessTables, resource)
+ return
+ }
+
if !isSystemResource(resource) {
searchPath := a.searchPath
if resource.Schema != "" {
@@ -66,6 +127,7 @@ func (a *accessTableExtractor) EnterQualified_name(qn *postgresql.Qualified_name
_, databaseMetadata, err := a.getDatabaseMetadata(a.ctx, a.instanceID, a.defaultDatabase)
if err != nil {
a.err = errors.Wrapf(err, "failed to get database metadata for database: %s", a.defaultDatabase)
+ return
}
// Access pseudo table or table/view we do not sync, return directly.
if databaseMetadata == nil {
diff --git a/backend/plugin/schema/pg/generate_migration.go b/backend/plugin/schema/pg/generate_migration.go
index bbb87bf85d..dfe7134bb2 100644
--- a/backend/plugin/schema/pg/generate_migration.go
+++ b/backend/plugin/schema/pg/generate_migration.go
@@ -1,7 +1,6 @@
package pg
import (
- "context"
"fmt"
"slices"
"strconv"
@@ -16,7 +15,6 @@ import (
pgpluginparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
"github.com/bytebase/bytebase/backend/plugin/schema"
"github.com/bytebase/bytebase/backend/plugin/schema/pg/ast"
- "github.com/bytebase/bytebase/backend/store/model"
)
func init() {
@@ -718,8 +716,10 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) error
for _, dep := range dependencies {
depID := getMigrationObjectID(dep.Schema, dep.Table)
- // Edge from dependency to dependent (table/view to view)
- graph.AddEdge(depID, viewID)
+ if allObjects[depID] {
+ // Edge from dependency to dependent (table/view to view)
+ graph.AddEdge(depID, viewID)
+ }
}
}
@@ -3427,18 +3427,17 @@ func (v *CreateOrReplaceVisitor) hasOrReplace(ctx *pgparser.CreatefunctionstmtCo
return false
}
-// getViewDependenciesFromAST extracts view dependencies from AST node
-// This reuses the same logic as getViewDependencies from get_database_metadata.go
-func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetadata *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {
+// getViewDependenciesFromAST extracts view dependencies from AST node using lightweight access table extraction.
+// This avoids the complexity and potential panics of GetQuerySpan by only extracting table/view references.
+// The caller is responsible for filtering dependencies against the set of objects being migrated.
+func getViewDependenciesFromAST(astNode any, schemaName string, _ *storepb.DatabaseSchemaMetadata) []*storepb.DependencyColumn {
if astNode == nil {
return []*storepb.DependencyColumn{}
}
- // Extract the SELECT statement from the VIEW AST node
var selectStatement string
if ctx, ok := astNode.(*pgparser.ViewstmtContext); ok {
- // Extract the SELECT statement from the ViewstmtContext
if ctx.Selectstmt() != nil {
// Try to get text using token stream first
if tokenStream := ctx.GetParser().GetTokenStream(); tokenStream != nil {
@@ -3449,7 +3448,7 @@ func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetada
}
}
- // Fallback to token-based approach if token stream approach failed
+ // Fallback to token-based approach if failed
if selectStatement == "" {
selectStatement = getTextFromAST(ctx.Selectstmt())
}
@@ -3460,60 +3459,39 @@ func getViewDependenciesFromAST(astNode any, schemaName string, fullSchemaMetada
return []*storepb.DependencyColumn{}
}
- // Use the same dependency extraction logic as getViewDependencies
queryStatement := strings.TrimSpace(selectStatement)
- // Use GetQuerySpan with the full schema metadata so it can resolve table/view references
- // Wrap in defer/recover to handle potential panics from incomplete metadata
- var span *base.QuerySpan
- var err error
- func() {
- defer func() {
- if r := recover(); r != nil {
- // If GetQuerySpan panics (e.g., due to incomplete metadata), treat it as an error
- err = errors.Errorf("GetQuerySpan panicked: %v", r)
- }
- }()
- span, err = pgpluginparser.GetQuerySpan(
- context.Background(),
- base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: func(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {
- // Return the full schema metadata so GetQuerySpan can resolve references
- dbMetadata := model.NewDatabaseMetadata(fullSchemaMetadata, false, false)
- return databaseName, dbMetadata, nil
- },
- ListDatabaseNamesFunc: func(_ context.Context, _ string) ([]string, error) {
- // Return empty list - we don't need actual database names for dependency extraction
- return []string{}, nil
- },
- },
- queryStatement,
- "", // database
- schemaName,
- false, // case sensitive
- )
- }()
-
- // If error parsing query span, return empty dependencies
+ accessTables, err := pgpluginparser.ExtractAccessTables(queryStatement, pgpluginparser.ExtractAccessTablesOption{
+ DefaultDatabase: "",
+ DefaultSchema: schemaName,
+ SkipMetadataValidation: true,
+ })
if err != nil {
- return []*storepb.DependencyColumn{} // nolint:nilerr
+ return []*storepb.DependencyColumn{}
}
- // Collect unique dependencies
+ // The caller will filter against allObjects
dependencyMap := make(map[string]*storepb.DependencyColumn)
- for sourceColumn := range span.SourceColumns {
- // Create dependency key in format: schema.table
- key := fmt.Sprintf("%s.%s", sourceColumn.Schema, sourceColumn.Table)
+ for _, resource := range accessTables {
+ if resource.Schema == "pg_catalog" || resource.Schema == "information_schema" {
+ continue
+ }
+
+ resourceSchema := resource.Schema
+ if resourceSchema == "" {
+ resourceSchema = schemaName
+ }
+
+ key := fmt.Sprintf("%s.%s", resourceSchema, resource.Table)
if _, exists := dependencyMap[key]; !exists {
dependencyMap[key] = &storepb.DependencyColumn{
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- Column: "*", // Use wildcard since we're tracking table-level dependencies
+ Schema: resourceSchema,
+ Table: resource.Table,
+ Column: "*", // Table-level dependencies
}
}
}
- // Convert map to slice
var dependencies []*storepb.DependencyColumn
for _, dep := range dependencyMap {
dependencies = append(dependencies, dep)
diff --git a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go b/backend/plugin/schema/pg/generate_migration_testcontainer_test.go
index 65dd7ff1c1..673a302552 100644
--- a/backend/plugin/schema/pg/generate_migration_testcontainer_test.go
+++ b/backend/plugin/schema/pg/generate_migration_testcontainer_test.go
@@ -2511,3 +2511,97 @@ CREATE TABLE table2 (
require.NoError(t, err)
require.True(t, fkExists, "FK constraint fk_table1_name should exist")
}
+
+// TestSDLViewDependencyChain tests that SDL migration correctly handles view dependencies
+// when a table is depended on by view1, and view1 is depended on by view2
+func TestSDLViewDependencyChain(t *testing.T) {
+ ctx := context.Background()
+
+ // Start PostgreSQL container
+ pgContainer, err := postgres.Run(ctx,
+ "postgres:16-alpine",
+ postgres.WithDatabase("test_db"),
+ postgres.WithUsername("postgres"),
+ postgres.WithPassword("test"),
+ testcontainers.WithWaitStrategy(
+ wait.ForLog("database system is ready to accept connections").
+ WithOccurrence(2).
+ WithStartupTimeout(5*time.Minute),
+ ),
+ )
+ require.NoError(t, err)
+ defer func() {
+ if err := pgContainer.Terminate(ctx); err != nil {
+ t.Logf("failed to terminate container: %s", err)
+ }
+ }()
+
+ // Get connection string
+ connectionString, err := pgContainer.ConnectionString(ctx, "sslmode=disable")
+ require.NoError(t, err)
+
+ // Connect to the database
+ connConfig, err := pgx.ParseConfig(connectionString)
+ require.NoError(t, err)
+ testDB := stdlib.OpenDB(*connConfig)
+ defer testDB.Close()
+
+ // This SDL text simulates the REAL multi-file scenario where files are merged alphabetically:
+ // Files: table_base.sql, test_view_0.sql, test_view2.sql
+ // Alphabetical order: table_base < test_view_0 < test_view2
+ // Dependency chain: table_base -> test_view2 -> test_view_0
+ // So SDL has: table_base, test_view_0 (depends on test_view2 which comes AFTER), test_view2
+ currentSDL := `
+CREATE TABLE table_base (
+ id serial,
+ info text not null,
+ created_at timestamp default current_timestamp,
+ updated_at timestamp default current_timestamp,
+ constraint pk_table_base_id primary key (id)
+);
+
+CREATE VIEW test_view_0 AS
+SELECT * FROM test_view2;
+
+CREATE VIEW test_view2 AS
+SELECT id, info, created_at, updated_at FROM table_base;
+`
+
+ previousSDL := ``
+
+ // Get current schema (empty database)
+ currentSchema, err := getSyncMetadataForGenerateMigration(ctx, connConfig, connConfig.Database)
+ require.NoError(t, err)
+ dbSchema := model.NewDatabaseSchema(currentSchema, nil, nil, storepb.Engine_POSTGRES, false)
+
+ // Get SDL diff
+ diff, err := GetSDLDiff(currentSDL, previousSDL, dbSchema, nil)
+ require.NoError(t, err)
+
+ // Generate migration SQL with topological sorting
+ migrationSQL, err := generateMigration(diff)
+ require.NoError(t, err)
+
+ // Execute the generated migration SQL
+ // This should work because generateMigration should have sorted objects by dependencies
+ _, err = testDB.Exec(migrationSQL)
+ require.NoError(t, err, "Migration SQL should execute successfully with correct view dependency order")
+
+ // Verify table was created
+ var tableExists bool
+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'table_base')").Scan(&tableExists)
+ require.NoError(t, err)
+ require.True(t, tableExists, "table_base should be created")
+
+ // Verify test_view2 was created (depends on table_base)
+ var view2Exists bool
+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.views WHERE table_name = 'test_view2')").Scan(&view2Exists)
+ require.NoError(t, err)
+ require.True(t, view2Exists, "test_view2 should be created")
+
+ // Verify test_view_0 was created (depends on test_view2)
+ var view0Exists bool
+ err = testDB.QueryRow("SELECT EXISTS (SELECT 1 FROM information_schema.views WHERE table_name = 'test_view_0')").Scan(&view0Exists)
+ require.NoError(t, err)
+ require.True(t, view0Exists, "test_view_0 should be created")
+}
diff --git a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go
index 4641b28bf6..04a40c551c 100644
--- a/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go
+++ b/backend/plugin/schema/pg/sdl_validation_testcontainer_test.go
@@ -1156,6 +1156,17 @@ func validateSchemaConsistency(schemaD, schemaC *storepb.DatabaseSchemaMetadata)
}
if len(diff.ViewChanges) > 0 {
diffDetails = append(diffDetails, fmt.Sprintf("ViewChanges: %d", len(diff.ViewChanges)))
+ for _, viewDiff := range diff.ViewChanges {
+ diffDetails = append(diffDetails, fmt.Sprintf(" View: %s.%s, Action: %v", viewDiff.SchemaName, viewDiff.ViewName, viewDiff.Action))
+ if viewDiff.OldView != nil && viewDiff.NewView != nil {
+ diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))
+ diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))
+ } else if viewDiff.OldView != nil {
+ diffDetails = append(diffDetails, fmt.Sprintf(" Old definition: %s", viewDiff.OldView.Definition))
+ } else if viewDiff.NewView != nil {
+ diffDetails = append(diffDetails, fmt.Sprintf(" New definition: %s", viewDiff.NewView.Definition))
+ }
+ }
}
if len(diff.MaterializedViewChanges) > 0 {
diffDetails = append(diffDetails, fmt.Sprintf("MaterializedViewChanges: %d", len(diff.MaterializedViewChanges)))