Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(masking)!: refuse PostgreSQL queries whose conditions read masked columns (#21651)
rebelice799732c12b1e
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.goindex 27672dc6f1..4bb66187ff 100644--- a/backend/api/v1/predicate_columns_check.go+++ b/backend/api/v1/predicate_columns_check.go@@ -2,7 +2,10 @@ package v1import ("context"+ "slices"+ "strings"+ "connectrpc.com/connect""github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"@@ -11,6 +14,32 @@ import ("github.com/bytebase/bytebase/backend/store")+// refuseSensitivePredicateColumns refuses, before execution, statements whose+// conditions read a column masked for user. The database evaluates conditions+// on unmasked values, so the rows it returns, their count, and the time it+// takes would reveal them.+func refuseSensitivePredicateColumns(ctx context.Context, stores *store.Store, spans []*base.QuerySpan, instance *store.InstanceMessage, user *store.UserMessage) error {+ if !isPredicateColumnsCheckEnabled(instance.Metadata.GetEngine()) {+ return nil+ }+ sensitiveColumns, err := NewQueryResultMasker(stores).ExtractSensitivePredicateColumns(ctx, spans, instance, user)+ if err != nil {+ return connect.NewError(connect.CodeInternal, errors.New(err.Error()))+ }+ for _, columns := range sensitiveColumns {+ if len(columns) == 0 {+ continue+ }+ names := make([]string, 0, len(columns))+ for _, column := range columns {+ names = append(names, column.String())+ }+ slices.Sort(names)+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("Using sensitive columns in query conditions is not allowed: %s", strings.Join(names, ", ")))+ }+ return nil+}+func (s *QueryResultMasker) ExtractSensitivePredicateColumns(ctx context.Context, spans []*base.QuerySpan, instance *store.InstanceMessage, user *store.UserMessage) ([][]base.ColumnResource, error) {var result [][]base.ColumnResource@@ -118,5 +147,5 @@ func (m *maskingLevelEvaluator) isSensitiveColumn(database *store.DatabaseMessag}func isPredicateColumnsCheckEnabled(engine storepb.Engine) bool {- return engine == storepb.Engine_MSSQL+ return engine == storepb.Engine_MSSQL || engine == storepb.Engine_POSTGRES}diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 34e69fdace..f9d0175537 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -602,14 +602,21 @@ func replaceBackupTableWithSource(ctx context.Context, stores *store.Store, inst}for _, span := range spans {- span.SourceColumns = generateNewSourceColumnSet(instance.Metadata.GetEngine(), span.SourceColumns, schema)- for _, result := range span.Results {- result.SourceColumns = generateNewSourceColumnSet(instance.Metadata.GetEngine(), result.SourceColumns, schema)- }+ remapBackupTableColumns(instance.Metadata.GetEngine(), span, schema)}return nil}+// remapBackupTableColumns points the columns span reads from backup tables at+// the source columns, whose masking policies apply to the backups too.+func remapBackupTableColumns(engine storepb.Engine, span *parserbase.QuerySpan, schema *model.SchemaMetadata) {+ span.SourceColumns = generateNewSourceColumnSet(engine, span.SourceColumns, schema)+ span.PredicateColumns = generateNewSourceColumnSet(engine, span.PredicateColumns, schema)+ for i := range span.Results {+ span.Results[i].SourceColumns = generateNewSourceColumnSet(engine, span.Results[i].SourceColumns, schema)+ }+}+func generateNewSourceColumnSet(engine storepb.Engine, origin parserbase.SourceColumnSet, schema *model.SchemaMetadata) parserbase.SourceColumnSet {result := make(parserbase.SourceColumnSet)for column := range origin {@@ -833,31 +840,12 @@ func queryRetry(schemaSyncer *schemasync.Syncer,multiStatement bool,) ([]*v1pb.QueryResult, []*parserbase.QuerySpan, time.Duration, error) {- var sensitivePredicateColumns [][]parserbase.ColumnResource// An explain request carries the bare statements the driver will wrap, so// their spans name the databases the plan describes.- spans, err := parserbase.GetQuerySpan(- ctx,- parserbase.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),- ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),- GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),- },- instance.Metadata.GetEngine(),- statements,- database.DatabaseName,- queryContext.Schema,- !store.IsObjectCaseSensitive(instance),- )+ spans, err := buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema)if err != nil {return nil, nil, time.Duration(0), err}- // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.- // If err != nil, this function will return the original spans.- if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {- slog.Debug("failed to replace backup table with source", log.BBError(err))- }if optionalAccessCheck != nil {// A smuggled write (EXPLAIN ANALYZE of a write) is refused earlier, in// the Query handler, before it reaches execution.@@ -866,20 +854,26 @@ func queryRetry(}slog.Debug("optional access check", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))}- if !queryContext.Explain && !queryContext.SkipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {- masker := NewQueryResultMasker(stores)- sensitivePredicateColumns, err = masker.ExtractSensitivePredicateColumns(ctx, spans, instance, user)- if err != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.New(err.Error()))- }- slog.Debug("extract sensitive predicate columns", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))- }-maskingEnabled := !queryContext.Explain && !queryContext.SkipMasking &&licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nilqueryContext.MaskingEnabled = maskingEnabledif maskingEnabled {+ // Columns a stale snapshot lacks can hide a condition from the predicate+ // check, so sync before execution. MaskResults refuses the result if the+ // columns stay unresolved. Permanently empty tables re-sync on every query+ // until BYT-10074 adds throttling.+ if unresolved := unresolvedColumnDatabases(spans, instance); len(unresolved) > 0 {+ if err := syncDatabaseMetadata(ctx, stores, schemaSyncer, instance, unresolved); err != nil {+ return nil, nil, time.Duration(0), err+ }+ if spans, err = buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema); err != nil {+ return nil, nil, time.Duration(0), err+ }+ }+ if err := refuseSensitivePredicateColumns(ctx, stores, spans, instance, user); err != nil {+ return nil, nil, time.Duration(0), err+ }if err := preExecuteMaskingCheck(ctx, stores, instance.Metadata.GetEngine(), database, spans); err != nil {return nil, nil, time.Duration(0), err}@@ -913,21 +907,7 @@ func queryRetry(syncDatabaseMap := make(map[string]bool)for i, r := range results {- if i >= len(spans) {- continue- }- // Re-sync even for error results: MaskResults also refuses partial rows.- // Permanently empty tables re-sync on every query until BYT-10074 adds throttling.- if maskingEnabled && maskingBlockedByUnresolvedColumns(spans[i], instance) {- for _, dbName := range spans[i].UnresolvedColumnsError.Databases() {- slog.Debug("database metadata need to sync: unresolved columns",- slog.String("instance", instance.ResourceID),- slog.String("database", dbName),- slog.String("detail", spans[i].UnresolvedColumnsError.Error()))- syncDatabaseMap[dbName] = true- }- }- if r.Error != "" {+ if i >= len(spans) || r.Error != "" {continue}if spans[i].NotFoundError != nil {@@ -938,55 +918,22 @@ func queryRetry(}}- // Sync database metadata.- for accessDatabaseName := range syncDatabaseMap {- slog.Debug("sync database metadata", slog.String("instance", instance.ResourceID), slog.String("database", accessDatabaseName))- d, err := stores.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID, DatabaseName: &accessDatabaseName})- if err != nil {- return nil, nil, duration, err- }- if d == nil {- // Leave the span error in place so masking can reject an untracked database.- slog.Debug("skip metadata sync: database not tracked",- slog.String("instance", instance.ResourceID),- slog.String("database", accessDatabaseName))- continue- }- if err := schemaSyncer.SyncDatabaseSchema(ctx, d); err != nil {- return nil, nil, duration, errors.Wrapf(err, "failed to sync database schema for database %q", accessDatabaseName)- }+ if err := syncDatabaseMetadata(ctx, stores, schemaSyncer, instance, syncDatabaseMap); err != nil {+ return nil, nil, duration, err}// Retry getting query span.if len(syncDatabaseMap) > 0 {slog.Debug("retry query after sync metadata", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))- spans, err = parserbase.GetQuerySpan(- ctx,- parserbase.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),- ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),- GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),- },- instance.Metadata.GetEngine(),- statements,- database.DatabaseName,- queryContext.Schema,- !store.IsObjectCaseSensitive(instance),- )+ spans, err = buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema)if err != nil {return nil, nil, time.Duration(0), err}- // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.- // If err != nil, this function will return the original spans.- if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {- slog.Debug("failed to replace backup table with source", log.BBError(err))- }- if !queryContext.SkipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {- masker := NewQueryResultMasker(stores)- sensitivePredicateColumns, err = masker.ExtractSensitivePredicateColumns(ctx, spans, instance, user)- if err != nil {- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.New(err.Error()))+ // A table the first span did not find may surface a condition the+ // pre-execution check could not see.+ if maskingEnabled {+ if err := refuseSensitivePredicateColumns(ctx, stores, spans, instance, user); err != nil {+ return nil, nil, duration, err}}}@@ -1017,33 +964,83 @@ func queryRetry(if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))}-- for i, result := range results {- if i >= len(sensitivePredicateColumns) {- continue- }- if len(sensitivePredicateColumns[i]) == 0 {- continue- }- result.Error = getSensitivePredicateColumnErrorMessages(sensitivePredicateColumns[i])- result.Rows = nil- result.RowsCount = 0- }}}return results, spans, duration, nil}-func getSensitivePredicateColumnErrorMessages(sensitiveColumns []parserbase.ColumnResource) string {- var buf bytes.Buffer- _, _ = buf.WriteString("Using sensitive columns in WHERE clause is not allowed: ")- for j, column := range sensitiveColumns {- if j > 0 {- _, _ = buf.WriteString(", ")+// buildQuerySpans analyzes statements for access checks and masking.+func buildQuerySpans(+ ctx context.Context,+ stores *store.Store,+ instance *store.InstanceMessage,+ database *store.DatabaseMessage,+ statements []parserbase.Statement,+ schema string,+) ([]*parserbase.QuerySpan, error) {+ spans, err := parserbase.GetQuerySpan(+ ctx,+ parserbase.GetQuerySpanContext{+ InstanceID: instance.ResourceID,+ GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),+ ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),+ GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),+ },+ instance.Metadata.GetEngine(),+ statements,+ database.DatabaseName,+ schema,+ !store.IsObjectCaseSensitive(instance),+ )+ if err != nil {+ return nil, err+ }+ // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.+ // If err != nil, this function will return the original spans.+ if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {+ slog.Debug("failed to replace backup table with source", log.BBError(err))+ }+ return spans, nil+}++// unresolvedColumnDatabases returns the databases whose stale metadata keeps+// masking from resolving a span's columns.+func unresolvedColumnDatabases(spans []*parserbase.QuerySpan, instance *store.InstanceMessage) map[string]bool {+ databases := make(map[string]bool)+ for _, span := range spans {+ if !maskingBlockedByUnresolvedColumns(span, instance) {+ continue+ }+ for _, name := range span.UnresolvedColumnsError.Databases() {+ slog.Debug("database metadata need to sync: unresolved columns",+ slog.String("instance", instance.ResourceID),+ slog.String("database", name),+ slog.String("detail", span.UnresolvedColumnsError.Error()))+ databases[name] = true+ }+ }+ return databases+}++func syncDatabaseMetadata(ctx context.Context, stores *store.Store, schemaSyncer *schemasync.Syncer, instance *store.InstanceMessage, databaseNames map[string]bool) error {+ for name := range databaseNames {+ slog.Debug("sync database metadata", slog.String("instance", instance.ResourceID), slog.String("database", name))+ d, err := stores.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID, DatabaseName: &name})+ if err != nil {+ return err+ }+ if d == nil {+ // Leave the span error in place so masking can reject an untracked database.+ slog.Debug("skip metadata sync: database not tracked",+ slog.String("instance", instance.ResourceID),+ slog.String("database", name))+ continue+ }+ if err := schemaSyncer.SyncDatabaseSchema(ctx, d); err != nil {+ return errors.Wrapf(err, "failed to sync database schema for database %q", name)}- _, _ = buf.WriteString(column.String())}- return buf.String()+ return nil}// queryRetryStopOnError runs the query and stops on encountering errors.diff --git a/backend/api/v1/sql_service_test.go b/backend/api/v1/sql_service_test.goindex 66fa49c32d..1f0058e5e2 100644--- a/backend/api/v1/sql_service_test.go+++ b/backend/api/v1/sql_service_test.go@@ -6,6 +6,7 @@ import ("testing""connectrpc.com/connect"+ metadatapb "github.com/bytebase/omni/metadata""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common"@@ -16,6 +17,7 @@ import (v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store"+ "github.com/bytebase/bytebase/backend/store/model")func TestSQLIAMDatabaseResourceUsesCanonicalInstanceScope(t *testing.T) {@@ -642,3 +644,27 @@ func TestExplainOfAWriteNamesItsTables(t *testing.T) {require.NoError(t, f.check(ctx, "[email protected]", true, "CALL p()", call))requireExplainDenied(t, f.check(ctx, "[email protected]", true, "CALL p()", call), permission.SQLExplain, "instances/test/databases/db_a")}++func TestRemapBackupTableColumns(t *testing.T) {+ dbMeta := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: common.BackupDatabaseNameOfEngine(storepb.Engine_POSTGRES),+ Tables: []*metadatapb.TableMetadata{{Name: "bak_t", Comment: "bytebase, source table (public, t)"}},+ }},+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)+ backup := parserbase.ColumnResource{Database: "db", Schema: common.BackupDatabaseNameOfEngine(storepb.Engine_POSTGRES), Table: "bak_t", Column: "secret"}+ source := parserbase.ColumnResource{Database: "db", Schema: "public", Table: "t", Column: "secret"}+ span := &parserbase.QuerySpan{+ SourceColumns: parserbase.SourceColumnSet{backup: true},+ PredicateColumns: parserbase.SourceColumnSet{backup: true},+ Results: []parserbase.QuerySpanResult{{Name: "secret", SourceColumns: parserbase.SourceColumnSet{backup: true}}},+ }++ remapBackupTableColumns(storepb.Engine_POSTGRES, span, getSchemaMetadata(storepb.Engine_POSTGRES, dbMeta))++ want := parserbase.SourceColumnSet{source: true}+ require.Equal(t, want, span.SourceColumns)+ require.Equal(t, want, span.PredicateColumns)+ require.Equal(t, want, span.Results[0].SourceColumns)+}diff --git a/backend/plugin/parser/pg/query_span.go b/backend/plugin/parser/pg/query_span.goindex fae50bcbed..6e5e458e41 100644--- a/backend/plugin/parser/pg/query_span.go+++ b/backend/plugin/parser/pg/query_span.go@@ -57,17 +57,20 @@ type omniQuerySpanExtractor struct {metaCache map[string]*model.DatabaseMetadatacat *catalog.CatalogfuncBodyCache map[uint32]*funcBodyAnalysis- // funcOrigDefs stores the original (non-stubbed) function definitions keyed- // by lowercase function name. Used by analyzeFunctionBody to get the real- // body when it was stubbed during catalog loading.- funcOrigDefs map[string]string+ // snapshotFuncs holds the snapshot's definition of every overload, keyed by+ // lowercase "schema.name". Used by analyzeFunctionBody for a function the+ // catalog installed as a stand-in, listed in standInFuncs by the same key;+ // the stand-in may stand for any overload of the name.+ snapshotFuncs map[string][]snapshotFunc+ standInFuncs map[string]bool// funcSourceColumns accumulates table-level access (column="") discovered inside// function bodies. Merged into the top-level QuerySpan.SourceColumns.funcSourceColumns base.SourceColumnSet- // funcPredicateColumns accumulates columns used in fallback WHERE/JOIN- // conditions. Function body analysis records predicates separately so callers- // can decide whether they enter the top-level QuerySpan.PredicateColumns.- funcPredicateColumns base.SourceColumnSet+ // predicateColumns accumulates the columns that decide which rows the+ // statement returns; it becomes QuerySpan.PredicateColumns. Function body+ // analysis collects into a set of its own, cached per function, and merges+ // it into the caller's.+ predicateColumns base.SourceColumnSet// fallbackCTEMap holds CTE definitions during fallback column extraction.// Set by extractFallbackColumns and used by extractColumnsFromRangeVar// to resolve CTE references.@@ -83,14 +86,15 @@ func newOmniQuerySpanExtractor(defaultDatabase string, searchPath []string, gCtxsearchPath = []string{"public"}}return &omniQuerySpanExtractor{- defaultDatabase: defaultDatabase,- searchPath: searchPath,- gCtx: gCtx,- metaCache: make(map[string]*model.DatabaseMetadata),- funcBodyCache: make(map[uint32]*funcBodyAnalysis),- funcOrigDefs: make(map[string]string),- funcSourceColumns: make(base.SourceColumnSet),- funcPredicateColumns: make(base.SourceColumnSet),+ defaultDatabase: defaultDatabase,+ searchPath: searchPath,+ gCtx: gCtx,+ metaCache: make(map[string]*model.DatabaseMetadata),+ funcBodyCache: make(map[uint32]*funcBodyAnalysis),+ snapshotFuncs: make(map[string][]snapshotFunc),+ standInFuncs: make(map[string]bool),+ funcSourceColumns: make(base.SourceColumnSet),+ predicateColumns: make(base.SourceColumnSet),}}@@ -127,6 +131,12 @@ func (e *omniQuerySpanExtractor) initCatalog() error {if err != nil {return errors.Wrapf(err, "failed to load the catalog for %q", e.defaultDatabase)}+ for key := range report.Degraded {+ if name, ok := strings.CutPrefix(key, "func:"); ok {+ name, _, _ = strings.Cut(name, "|")+ e.standInFuncs[strings.ToLower(name)] = true+ }+ }if len(report.Degraded) > 0 || len(report.Missing) > 0 {slog.Debug("catalog loaded with stand-ins",slog.String("database", e.defaultDatabase),@@ -135,16 +145,15 @@ func (e *omniQuerySpanExtractor) initCatalog() error {)}- // Populate original (non-stubbed) function definitions for PL/pgSQL body- // analysis.for _, s := range meta.GetProto().GetSchemas() {for _, f := range s.GetFunctions() {if f.Definition == "" {continue}- origBody := extractFuncBodyFromDef(f.Definition)- if origBody != "" {- e.funcOrigDefs[strings.ToLower(f.Name)] = origBody+ _, body, language := parseFuncDefForLineage(f.Definition)+ if body != "" {+ key := strings.ToLower(s.Name + "." + f.Name)+ e.snapshotFuncs[key] = append(e.snapshotFuncs[key], snapshotFunc{language: language, body: body})}}}@@ -285,13 +294,27 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)queryType, isExplainAnalyze := classifyQueryType(omniStmts[0].AST, allSystems)// Step 5: Return early for non-SELECT queries, keeping the tables the- // statement touches and no result columns.+ // statement touches and no result columns. EXPLAIN ANALYZE runs its query+ // and reports the rows each condition kept, so its conditions still count+ // as predicates.if queryType != base.Select || isExplainAnalyze {+ var unresolved *base.UnresolvedColumnsError+ if isExplainAnalyze {+ if err := e.collectExplainedPredicateColumns(omniStmts[0].AST, accessesMap); err != nil {+ return nil, err+ }+ e.mergeFunctionAccesses(accessesMap)+ // Columns a stale snapshot lacks can hide a condition, so the+ // caller syncs before running it, as for the SELECT itself.+ unresolved = e.unresolvedColumnsError(accessesMap)+ }return &base.QuerySpan{- Type: queryType,- ExplainAnalyze: isExplainAnalyze,- SourceColumns: accessesMap,- Results: []base.QuerySpanResult{},+ Type: queryType,+ ExplainAnalyze: isExplainAnalyze,+ SourceColumns: accessesMap,+ PredicateColumns: e.predicateColumns,+ Results: []base.QuerySpanResult{},+ UnresolvedColumnsError: unresolved,}, nil}@@ -310,26 +333,24 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)return nil, errors.Wrapf(err, "failed to initialize catalog")}- query, err := e.cat.AnalyzeSelectStmt(selStmt)+ query, err := e.analyzeSelect(selStmt)if err != nil {+ e.collectFallbackPredicateColumns(selStmt, accessesMap)// Before falling back, try to handle user-defined table-returning functions// that omni's AnalyzeSelectStmt can't resolve (e.g., RETURNS TABLE functions// used as table sources: SELECT * FROM func()).- if results := e.tryUserFuncTableSource(selStmt, accessesMap); results != nil {- return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessesMap,- Results: results,- UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),- }, nil- }- // Fail-open: return access tables with best-effort column names and lineage- // when analysis fails (e.g., unsupported built-in functions).+ results := e.tryUserFuncTableSource(selStmt, accessesMap)+ if results == nil {+ // Fail-open: return access tables with best-effort column names and lineage+ // when analysis fails (e.g., unsupported built-in functions).+ results = e.extractFallbackColumns(selStmt)+ }+ e.mergeFunctionAccesses(accessesMap)return &base.QuerySpan{Type: base.Select,SourceColumns: accessesMap,- Results: e.extractFallbackColumns(selStmt),- PredicateColumns: e.funcPredicateColumns,+ Results: results,+ PredicateColumns: e.predicateColumns,UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),}, nil}@@ -338,21 +359,63 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)// SourceColumns (table-level access) is already captured in accessesMap// from ExtractAccessTables above — no need to re-extract from the query.results := e.extractLineage(query, selStmt)+ e.collectStatementPredicateColumns(query, selStmt, accessesMap)- // Merge columns discovered inside function bodies into the top-level sets.- for col := range e.funcSourceColumns {- accessesMap[col] = true- }+ e.mergeFunctionAccesses(accessesMap)return &base.QuerySpan{Type: base.Select,SourceColumns: accessesMap,- PredicateColumns: e.funcPredicateColumns,+ PredicateColumns: e.predicateColumns,Results: results,UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),}, nil}+func (e *omniQuerySpanExtractor) collectExplainedPredicateColumns(stmt ast.Node, accesses base.SourceColumnSet) error {+ explain, ok := stmt.(*ast.ExplainStmt)+ if !ok {+ return nil+ }+ explained, ok := explain.Query.(*ast.SelectStmt)+ if !ok {+ return nil+ }+ if err := e.initCatalog(); err != nil {+ return errors.Wrapf(err, "failed to initialize catalog")+ }+ if query, err := e.analyzeSelect(explained); err == nil {+ e.collectStatementPredicateColumns(query, explained, accesses)+ } else {+ e.collectFallbackPredicateColumns(explained, accesses)+ }+ return nil+}++// mergeFunctionAccesses adds the tables the analyzed function bodies read to+// accesses, so access checks and the unresolved-columns signal see them.+func (e *omniQuerySpanExtractor) mergeFunctionAccesses(accesses base.SourceColumnSet) {+ for col := range e.funcSourceColumns {+ accesses[col] = true+ }+}++// analyzeSelect analyzes sel for lineage. A function the synced schema does not+// define, such as one from an extension, stays an opaque call over its+// arguments instead of failing the statement into the parse-tree fallback.+func (e *omniQuerySpanExtractor) analyzeSelect(sel *ast.SelectStmt) (*catalog.Query, error) {+ return e.cat.AnalyzeSelectStmtWithOptions(sel, catalog.AnalyzeOptions{OpaqueUnknownFunctions: true})+}++// outputColumn returns the target entry of q's attNum-th output column. Junk+// entries follow the output columns, so the position indexes the target list.+func outputColumn(q *catalog.Query, attNum int) *catalog.TargetEntry {+ if q == nil || attNum < 1 || attNum > len(q.TargetList) {+ return nil+ }+ return q.TargetList[attNum-1]+}+// extractLineage extracts the result column lineage from an analyzed query.// selStmt is the original parse tree, used to determine IsPlainField// (which is true only for columns that came from * expansion).@@ -465,20 +528,19 @@ func isUltimatelyPlainColumnWithVisited(q *catalog.Query, expr catalog.AnalyzedEreturn false}rte := q.RangeTable[v.RangeIdx]- colIdx := int(v.AttNum - 1)switch rte.Kind {case catalog.RTERelation:return truecase catalog.RTESubquery:- if rte.Subquery != nil && colIdx >= 0 && colIdx < len(rte.Subquery.TargetList) {- return isUltimatelyPlainColumnWithVisited(rte.Subquery, rte.Subquery.TargetList[colIdx].Expr, visited)+ if te := outputColumn(rte.Subquery, int(v.AttNum)); te != nil {+ return isUltimatelyPlainColumnWithVisited(rte.Subquery, te.Expr, visited)}case catalog.RTECTE:if rte.CTEIndex >= 0 && rte.CTEIndex < len(q.CTEList) {cte := q.CTEList[rte.CTEIndex]- if cte.Query != nil && colIdx >= 0 && colIdx < len(cte.Query.TargetList) {- return isUltimatelyPlainColumnWithVisited(cte.Query, cte.Query.TargetList[colIdx].Expr, visited)+ if te := outputColumn(cte.Query, int(v.AttNum)); te != nil {+ return isUltimatelyPlainColumnWithVisited(cte.Query, te.Expr, visited)}}default:@@ -696,7 +758,7 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Querye.walkExprWithVisited(queryStack, arg, result, visited)}// Trace lineage through user-defined function body.- if proc := e.cat.GetUserProcByOID(v.FuncOID); proc != nil {+ for _, proc := range e.userProcsCalled(v) {// Expression context: merge all output columns into one set.for _, colSet := range e.analyzeFunctionBody(proc) {for k, val := range colSet {@@ -704,28 +766,6 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Query}}}- case *catalog.AggExpr:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)- }- case *catalog.OpExpr:- e.walkExprWithVisited(queryStack, v.Left, result, visited)- e.walkExprWithVisited(queryStack, v.Right, result, visited)- case *catalog.BoolExprQ:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)- }- case *catalog.CaseExprQ:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- for _, w := range v.When {- e.walkExprWithVisited(queryStack, w.Condition, result, visited)- e.walkExprWithVisited(queryStack, w.Result, result, visited)- }- e.walkExprWithVisited(queryStack, v.Default, result, visited)- case *catalog.CoalesceExprQ:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)- }case *catalog.SubLinkExpr:e.walkExprWithVisited(queryStack, v.TestExpr, result, visited)if v.SubQuery != nil {@@ -736,47 +776,21 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Query}}}- case *catalog.RelabelExpr:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- case *catalog.CoerceViaIOExpr:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- case *catalog.RowExprQ:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)- }- case *catalog.WindowFuncExpr:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)+ case *catalog.SetOpColumnExpr:+ // q is the set operation. EXCEPT returns only rows of its left branch.+ branches := []*catalog.Query{q.LArg, q.RArg}+ if q.SetOp == catalog.SetOpExcept || q.SetOp == catalog.SetOpExceptAll {+ branches = branches[:1]}- e.walkExprWithVisited(queryStack, v.AggFilter, result, visited)- case *catalog.NullIfExprQ:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)- }- case *catalog.MinMaxExprQ:- for _, arg := range v.Args {- e.walkExprWithVisited(queryStack, arg, result, visited)+ for _, branch := range branches {+ if te := outputColumn(branch, int(v.ResNo)); te != nil {+ e.walkExprWithVisited(appendQuery(queryStack, branch), te.Expr, result, visited)+ }}- case *catalog.ArrayExprQ:- for _, elem := range v.Elements {- e.walkExprWithVisited(queryStack, elem, result, visited)- }- case *catalog.ScalarArrayOpExpr:- e.walkExprWithVisited(queryStack, v.Left, result, visited)- e.walkExprWithVisited(queryStack, v.Right, result, visited)- case *catalog.CollateExprQ:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- case *catalog.NullTestExpr:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- case *catalog.BooleanTestExpr:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- case *catalog.DistinctExprQ:- e.walkExprWithVisited(queryStack, v.Left, result, visited)- e.walkExprWithVisited(queryStack, v.Right, result, visited)- case *catalog.FieldSelectExprQ:- e.walkExprWithVisited(queryStack, v.Arg, result, visited)- // ConstExpr, SQLValueFuncExpr, CoerceToDomainValueExpr — no column refs.default:+ for _, child := range catalog.AnalyzedExprChildren(expr) {+ e.walkExprWithVisited(queryStack, child, result, visited)+ }}}@@ -840,32 +854,34 @@ func (e *omniQuerySpanExtractor) resolveVar(queryStack []*catalog.Query, v *cata}] = truecase catalog.RTESubquery:- if rte.Subquery == nil {- return- }- if colIdx >= 0 && colIdx < len(rte.Subquery.TargetList) {- te := rte.Subquery.TargetList[colIdx]+ if te := outputColumn(rte.Subquery, int(v.AttNum)); te != nil {e.walkExprWithVisited(appendQuery(effectiveStack, rte.Subquery), te.Expr, result, visited)}case catalog.RTECTE:- if cte := findCTEForRangeTableEntry(effectiveStack, rte); cte != nil && cte.Query != nil && colIdx >= 0 && colIdx < len(cte.Query.TargetList) {- te := cte.Query.TargetList[colIdx]- e.walkExprWithVisited(appendQuery(effectiveStack, cte.Query), te.Expr, result, visited)+ if cte := findCTEForRangeTableEntry(effectiveStack, rte); cte != nil {+ if te := outputColumn(cte.Query, int(v.AttNum)); te != nil {+ e.walkExprWithVisited(appendQuery(effectiveStack, cte.Query), te.Expr, result, visited)+ }}case catalog.RTEJoin:- // After analysis, VarExprs typically point to base RTEs, not join RTEs.+ // Only a column that FULL JOIN USING merges is read through the join.+ if colIdx >= 0 && colIdx < len(rte.JoinAliasVars) {+ e.walkExprWithVisited(effectiveStack, rte.JoinAliasVars[colIdx], result, visited)+ }case catalog.RTEFunction:// Trace lineage through user-defined function body.if len(rte.FuncExprs) > 0 {if fc, ok := rte.FuncExprs[0].(*catalog.FuncCallExpr); ok {- if proc := e.cat.GetUserProcByOID(fc.FuncOID); proc != nil {- bodySets := e.analyzeTableFunctionBody(proc)- if colIdx >= 0 && colIdx < len(bodySets) {- for k, val := range bodySets[colIdx] {- result[k] = val+ if procs := e.userProcsCalled(fc); len(procs) > 0 {+ for _, proc := range procs {+ bodySets := e.analyzeFunctionBody(proc)+ if colIdx >= 0 && colIdx < len(bodySets) {+ for k, val := range bodySets[colIdx] {+ result[k] = val+ }}}} else {@@ -962,7 +978,7 @@ func (e *omniQuerySpanExtractor) extractFallbackColumns(selStmt *ast.SelectStmt)whereColSet := make(base.SourceColumnSet)analyzer.extractColumnRefsFromExpr(selStmt.WhereClause, fromTables, whereColSet)for k, v := range whereColSet {- e.funcPredicateColumns[k] = v+ e.predicateColumns[k] = v}}return results@@ -1368,7 +1384,7 @@ func (e *omniQuerySpanExtractor) extractColumnsFromRangeFunction(rf *ast.RangeFuif proc := e.lookupUserProcByName(funcName); proc != nil {outNames := getOutputParamNames(proc)if len(outNames) > 0 {- bodySets := e.analyzeTableFunctionBody(proc)+ bodySets := e.analyzeFunctionBody(proc)var results []base.QuerySpanResultfor i, name := range outNames {colSet := make(base.SourceColumnSet)@@ -1530,7 +1546,7 @@ func (e *omniQuerySpanExtractor) tryUserFuncTableSource(selStmt *ast.SelectStmt,if len(outNames) == 0 {continue}- bodySets := e.analyzeTableFunctionBody(proc)+ bodySets := e.analyzeFunctionBody(proc)return e.buildFuncResults(outNames, bodySets, accessesMap)}@@ -1677,7 +1693,7 @@ func (e *omniQuerySpanExtractor) tryMetadataFuncLookup(funcName string, accesses// Use a temporary OID for caching.syntheticProc.OID = uint32(0xFFFF0000 + len(e.funcBodyCache))- bodySets := e.analyzeTableFunctionBody(syntheticProc)+ bodySets := e.analyzeFunctionBody(syntheticProc)return e.buildFuncResults(outNames, bodySets, accessesMap)}diff --git a/backend/plugin/parser/pg/query_span_plpgsql.go b/backend/plugin/parser/pg/query_span_plpgsql.goindex 34ed565fcf..315aa92115 100644--- a/backend/plugin/parser/pg/query_span_plpgsql.go+++ b/backend/plugin/parser/pg/query_span_plpgsql.go@@ -13,6 +13,13 @@ import ("github.com/bytebase/bytebase/backend/plugin/parser/base")+// snapshotFunc is a function's language and body as the schema snapshot+// defines it.+type snapshotFunc struct {+ language string+ body string+}+type funcBodyAnalysis struct {sourceColumns []base.SourceColumnSetpredicateColumns base.SourceColumnSet@@ -25,13 +32,7 @@ type funcBodyAnalysis struct {func (e *omniQuerySpanExtractor) analyzeFunctionBody(proc *catalog.UserProc) []base.SourceColumnSet {analysis := e.analyzeFunctionBodyForProc(proc)e.mergeFunctionSourceTables(analysis.sourceColumns)- mergeSourceColumnSetInto(e.funcPredicateColumns, analysis.predicateColumns)- return analysis.sourceColumns-}--func (e *omniQuerySpanExtractor) analyzeTableFunctionBody(proc *catalog.UserProc) []base.SourceColumnSet {- analysis := e.analyzeFunctionBodyForProc(proc)- e.mergeFunctionSourceTables(analysis.sourceColumns)+ mergeSourceColumnSetInto(e.predicateColumns, analysis.predicateColumns)return analysis.sourceColumns}@@ -50,38 +51,51 @@ func (e *omniQuerySpanExtractor) analyzeFunctionBodyForProc(proc *catalog.UserPr// Store nil sentinel to detect recursion.e.funcBodyCache[proc.OID] = nil- outerPredicateColumns := e.funcPredicateColumns+ outerPredicateColumns := e.predicateColumnsbodyPredicateColumns := make(base.SourceColumnSet)- e.funcPredicateColumns = bodyPredicateColumns+ e.predicateColumns = bodyPredicateColumnsdefer func() {- e.funcPredicateColumns = outerPredicateColumns+ e.predicateColumns = outerPredicateColumns}()- // If the function body was stubbed during catalog loading (to avoid type- // validation errors), use the original body from metadata instead.- body := proc.Body- if origBody, ok := e.funcOrigDefs[strings.ToLower(proc.Name)]; ok && origBody != "" {- body = origBody+ // The catalog keeps each overload's own body. A function it could not+ // install is a stand-in with a placeholder body, and the snapshot cannot+ // tell which overload it stands for, so every overload's body is analyzed+ // and merged.+ defs := []snapshotFunc{{language: proc.Language, body: proc.Body}}+ if proc.Schema != nil {+ key := strings.ToLower(proc.Schema.Name + "." + proc.Name)+ if e.standInFuncs[key] && len(e.snapshotFuncs[key]) > 0 {+ defs = e.snapshotFuncs[key]+ }}var result []base.SourceColumnSetvar err error-- switch strings.ToLower(proc.Language) {- case "plpgsql":- result, err = e.analyzePLpgSQLBody(proc, body)- case "sql":- result, err = e.analyzeSQLBody(body)- default:- // Unsupported language (C, internal, etc.) — unknown lineage.- result = makeEmptySets(proc)+ for _, def := range defs {+ defResult, defErr := e.analyzeBodyAs(proc, def)+ if defErr != nil {+ err = defErr+ continue+ }+ if len(defs) == 1 {+ result = defResult+ break+ }+ for i, set := range defResult {+ if i >= len(result) {+ result = append(result, make(base.SourceColumnSet))+ }+ mergeSourceColumnSetInto(result[i], set)+ }}- if err != nil {- // On error, cache empty sets so we don't retry.+ if err != nil && result == nil {+ // On error, cache empty sets so we don't retry. The conditions found+ // before the error still run, so they stay.e.funcBodyCache[proc.OID] = &funcBodyAnalysis{sourceColumns: makeEmptySets(proc),- predicateColumns: make(base.SourceColumnSet),+ predicateColumns: cloneSourceColumnSet(bodyPredicateColumns),}return e.funcBodyCache[proc.OID]}@@ -93,6 +107,18 @@ func (e *omniQuerySpanExtractor) analyzeFunctionBodyForProc(proc *catalog.UserPrreturn e.funcBodyCache[proc.OID]}+func (e *omniQuerySpanExtractor) analyzeBodyAs(proc *catalog.UserProc, def snapshotFunc) ([]base.SourceColumnSet, error) {+ switch strings.ToLower(def.language) {+ case "plpgsql":+ return e.analyzePLpgSQLBody(proc, def.body)+ case "sql":+ return e.analyzeSQLBody(def.body)+ default:+ // Unsupported language (C, internal, etc.) — unknown lineage.+ return makeEmptySets(proc), nil+ }+}+func cloneSourceColumnSet(src base.SourceColumnSet) base.SourceColumnSet {dst := make(base.SourceColumnSet)for k, v := range src {@@ -152,30 +178,64 @@ func (e *omniQuerySpanExtractor) analyzeSQLBody(body string) ([]base.SourceColumreturn nil, errors.Wrapf(err, "failed to parse SQL function body")}- // Find the last SELECT statement (SQL functions return the result of the last statement).- for i := len(stmts) - 1; i >= 0; i-- {- selStmt, ok := stmts[i].AST.(*ast.SelectStmt)+ // Every statement in the body runs, so each one's conditions count; the+ // last one returns the function's result.+ last := -1+ for i, stmt := range stmts {+ if stmt.AST != nil {+ last = i+ }+ }+ var result []base.SourceColumnSet+ for i, stmt := range stmts {+ if stmt.AST == nil {+ continue+ }+ accesses := e.accessTablesOf(stmt.Text)+ // The caller reads what the body reads, whether or not the result shows it.+ mergeSourceColumnSetInto(e.funcSourceColumns, accesses)+ selStmt, ok := stmt.AST.(*ast.SelectStmt)if !ok {+ // The analyzer reads SELECT alone, so another statement's conditions+ // and RETURNING list are matched by name.+ e.collectFallbackPredicateColumns(stmt.AST, accesses)+ returning := e.returningLineage(stmt.AST, accesses)+ if i == last {+ result = returning+ } else {+ for _, set := range returning {+ mergeSourceColumnSetInto(e.predicateColumns, set)+ }+ }continue}- query, err := e.cat.AnalyzeSelectStmt(selStmt)+ query, err := e.analyzeSelect(selStmt)if err != nil {+ e.collectFallbackPredicateColumns(selStmt, accesses)+ if i != last {+ e.collectDiscardedOutputColumns(nil, selStmt, accesses)+ continue+ }// Fall back to parse-tree extraction when analysis fails// (e.g., nested function calls, parameter references).fallbackResults := e.extractFallbackColumns(selStmt)- if len(fallbackResults) > 0 {- result := make([]base.SourceColumnSet, len(fallbackResults))- for j, fr := range fallbackResults {- result[j] = fr.SourceColumns- }- return result, nil+ if len(fallbackResults) == 0 {+ return nil, errors.Wrapf(err, "failed to analyze SQL function body")}- return nil, errors.Wrapf(err, "failed to analyze SQL function body")+ result = make([]base.SourceColumnSet, len(fallbackResults))+ for j, fr := range fallbackResults {+ result[j] = fr.SourceColumns+ }+ continue+ }+ e.collectStatementPredicateColumns(query, selStmt, accesses)+ if i == last {+ result = e.extractFuncLineage(query)+ } else {+ e.collectDiscardedOutputColumns(query, selStmt, accesses)}- return e.extractFuncLineage(query), nil}-- return nil, nil+ return result, nil}func (e *omniQuerySpanExtractor) extractFuncLineage(q *catalog.Query) []base.SourceColumnSet {@@ -306,9 +366,14 @@ func (a *plpgsqlAnalyzer) walkBlock(block *plpgsqlast.PLBlock) {for _, decl := range block.Declarations {switch d := decl.(type) {case *plpgsqlast.PLDeclare:- a.scope.set(d.Name, make(base.SourceColumnSet))+ sources := make(base.SourceColumnSet)+ if d.Default != "" {+ sources = a.analyzeEmbeddedExpr(d.Default)+ }+ a.scope.set(d.Name, sources)case *plpgsqlast.PLCursorDecl:- a.scope.set(d.Name, make(base.SourceColumnSet))+ // A cursor carries the columns of its query to what FETCH reads.+ a.scope.set(d.Name, a.analyzeQuerySources(d.Query))case *plpgsqlast.PLAliasDecl:// Alias references another variable — copy its sources.if sources, ok := a.scope.get(d.RefName); ok {@@ -356,16 +421,23 @@ func (a *plpgsqlAnalyzer) walkStmt(node plpgsqlast.Node) {a.walkStmt(s)}case *plpgsqlast.PLWhile:+ a.handleCondition(stmt.Condition)for _, s := range stmt.Body {a.walkStmt(s)}case *plpgsqlast.PLForI:+ a.handleCondition(stmt.Lower)+ a.handleCondition(stmt.Upper)+ a.handleCondition(stmt.Step)for _, s := range stmt.Body {a.walkStmt(s)}case *plpgsqlast.PLForS:a.handleForS(stmt)case *plpgsqlast.PLForC:+ if sources, ok := a.scope.get(stmt.CursorVar); ok {+ a.scope.merge(stmt.Var, sources)+ }for _, s := range stmt.Body {a.walkStmt(s)}@@ -374,18 +446,124 @@ func (a *plpgsqlAnalyzer) walkStmt(node plpgsqlast.Node) {a.walkStmt(s)}case *plpgsqlast.PLForEachA:+ a.handleCondition(stmt.ArrayExpr)for _, s := range stmt.Body {a.walkStmt(s)}case *plpgsqlast.PLBlock:a.walkBlock(stmt)+ // PERFORM discards its rows but still runs, so how long it takes reveals+ // what it reads.+ case *plpgsqlast.PLPerform:+ a.handleCondition(stmt.Expr)+ case *plpgsqlast.PLDynExecute:+ a.handleDynExecute(stmt)+ case *plpgsqlast.PLOpen:+ if stmt.Query != "" {+ a.scope.merge(stmt.CursorVar, a.analyzeQuerySources(stmt.Query))+ }+ case *plpgsqlast.PLFetch:+ if sources, ok := a.scope.get(stmt.CursorVar); ok {+ for _, varName := range stmt.Into {+ a.scope.merge(varName, sources)+ }+ }+ case *plpgsqlast.PLCall:+ a.handleCall(stmt)+ // RAISE puts the values of its parameters in the message the caller sees.+ case *plpgsqlast.PLRaise:+ for _, param := range stmt.Params {+ a.handleCondition(param)+ }+ case *plpgsqlast.PLExit:+ a.handleCondition(stmt.Condition)+ case *plpgsqlast.PLAssert:+ a.handleCondition(stmt.Condition)// Skip nodes that don't affect lineage:- // PLDynExecute, PLRaise, PLNull, PLCommit, PLRollback, PLPerform,- // PLCall, PLOpen, PLFetch, PLClose, PLGetDiag, PLAssert, PLExit+ // PLNull, PLCommit, PLRollback, PLClose, PLGetDiagdefault:}}+// handleCondition records the columns an expression that steers the body+// reads, directly or through variables: it decides which statements run, and+// so what the function returns and how long it takes.+func (a *plpgsqlAnalyzer) handleCondition(expr string) {+ if strings.TrimSpace(expr) == "" {+ return+ }+ mergeSourceColumnSetInto(a.extractor.predicateColumns, a.analyzeEmbeddedExpr(expr))+ stmts, err := ParsePg("SELECT " + expr)+ if err != nil || len(stmts) != 1 {+ return+ }+ a.addScopeReferences(stmts[0].AST)+}++// addScopeReferences records, as predicates, the columns that the variables+// node reads carry.+func (a *plpgsqlAnalyzer) addScopeReferences(node ast.Node) {+ mergeSourceColumnSetInto(a.extractor.predicateColumns, a.scopeSources(node))+}++// scopeSources returns the columns that the variables node reads carry. A+// variable, or the record variable a qualified reference such as r.ssn names,+// carries the columns it was assigned from.+func (a *plpgsqlAnalyzer) scopeSources(node ast.Node) base.SourceColumnSet {+ sources := make(base.SourceColumnSet)+ ast.Inspect(node, func(n ast.Node) bool {+ if ref, ok := n.(*ast.ColumnRef); ok && ref.Fields != nil && len(ref.Fields.Items) > 0 {+ if name, ok := ref.Fields.Items[0].(*ast.String); ok {+ if varSources, ok := a.scope.get(name.Str); ok {+ mergeSourceColumnSetInto(sources, varSources)+ }+ }+ }+ return true+ })+ return sources+}++// analyzeQuerySources returns every column the output of sql reads, merged.+func (a *plpgsqlAnalyzer) analyzeQuerySources(sql string) base.SourceColumnSet {+ merged := make(base.SourceColumnSet)+ if strings.TrimSpace(sql) == "" {+ return merged+ }+ for _, set := range a.analyzeEmbeddedSQL(sql) {+ mergeSourceColumnSetInto(merged, set)+ }+ return merged+}++// handleCall analyzes the body of the procedure CALL invokes, whatever+// overload it is. The procedure runs for its effects, so the values passed to+// it count as predicates.+func (a *plpgsqlAnalyzer) handleCall(stmt *plpgsqlast.PLCall) {+ stmts, err := ParsePg(stmt.SQLText)+ if err != nil || len(stmts) != 1 {+ return+ }+ call, ok := stmts[0].AST.(*ast.CallStmt)+ if !ok || call.Funccall == nil {+ return+ }+ for _, proc := range a.extractor.userProcsNamed(astStrings(call.Funccall.Funcname)) {+ a.extractor.analyzeFunctionBody(proc)+ }+ if call.Funccall.Args == nil {+ return+ }+ for _, arg := range call.Funccall.Args.Items {+ loc := ast.NodeLoc(arg)+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(stmt.SQLText) {+ a.handleCondition(stmt.SQLText[loc.Start:loc.End])+ continue+ }+ a.addScopeReferences(arg)+ }+}+func (a *plpgsqlAnalyzer) handleAssign(stmt *plpgsqlast.PLAssign) {varName := extractBaseVarName(stmt.Target)sources := a.analyzeEmbeddedExpr(stmt.Expr)@@ -419,10 +597,20 @@ func (a *plpgsqlAnalyzer) handleReturnNext(stmt *plpgsqlast.PLReturnNext) {}func (a *plpgsqlAnalyzer) handleExecSQL(stmt *plpgsqlast.PLExecSQL) {- if len(stmt.Into) == 0 {+ stmts, err := ParsePg(stmt.SQLText)+ if err != nil || len(stmts) != 1 {return}- results := a.analyzeEmbeddedSQL(stmt.SQLText)+ var results []base.SourceColumnSet+ if _, ok := stmts[0].AST.(*ast.SelectStmt); ok {+ results = a.analyzeEmbeddedSQL(stmt.SQLText)+ } else {+ // The analyzer reads SELECT alone, so another statement, which runs for+ // its effect, has its conditions and RETURNING list matched by name.+ accesses := a.extractor.accessTablesOf(stmt.SQLText)+ a.extractor.collectFallbackPredicateColumns(stmts[0].AST, accesses)+ results = a.extractor.returningLineage(stmts[0].AST, accesses)+ }for i, varName := range stmt.Into {if i < len(results) {a.scope.merge(varName, results[i])@@ -430,6 +618,55 @@ func (a *plpgsqlAnalyzer) handleExecSQL(stmt *plpgsqlast.PLExecSQL) {}}+// handleDynExecute analyzes EXECUTE of a constant query like the statement it+// runs. A query built at run time cannot be analyzed.+func (a *plpgsqlAnalyzer) handleDynExecute(stmt *plpgsqlast.PLDynExecute) {+ stmts, err := ParsePg("SELECT " + stmt.Query)+ if err != nil || len(stmts) != 1 {+ return+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok || sel.TargetList == nil || len(sel.TargetList.Items) != 1 {+ return+ }+ rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)+ if !ok {+ return+ }+ query, ok := constantString(rt.Val)+ if !ok {+ return+ }+ a.handleExecSQL(&plpgsqlast.PLExecSQL{SQLText: query, Into: stmt.Into})+}++// constantString returns the text of a string literal, a cast of one, or a+// concatenation of them.+func constantString(node ast.Node) (string, bool) {+ switch v := node.(type) {+ case *ast.A_Const:+ if s, ok := v.Val.(*ast.String); ok {+ return s.Str, true+ }+ case *ast.TypeCast:+ return constantString(v.Arg)+ case *ast.A_Expr:+ if names := astStrings(v.Name); len(names) == 1 && names[0] == "||" {+ left, ok := constantString(v.Lexpr)+ if !ok {+ return "", false+ }+ right, ok := constantString(v.Rexpr)+ if !ok {+ return "", false+ }+ return left + right, true+ }+ default:+ }+ return "", false+}+func (a *plpgsqlAnalyzer) handleForS(stmt *plpgsqlast.PLForS) {results := a.analyzeEmbeddedSQL(stmt.Query)merged := make(base.SourceColumnSet)@@ -445,11 +682,13 @@ func (a *plpgsqlAnalyzer) handleForS(stmt *plpgsqlast.PLForS) {}func (a *plpgsqlAnalyzer) handleIf(stmt *plpgsqlast.PLIf) {+ a.handleCondition(stmt.Condition)for _, s := range stmt.ThenBody {a.walkStmt(s)}for _, elsif := range stmt.ElsIfs {if ei, ok := elsif.(*plpgsqlast.PLElsIf); ok {+ a.handleCondition(ei.Condition)for _, s := range ei.Body {a.walkStmt(s)}@@ -461,8 +700,10 @@ func (a *plpgsqlAnalyzer) handleIf(stmt *plpgsqlast.PLIf) {}func (a *plpgsqlAnalyzer) handleCase(stmt *plpgsqlast.PLCase) {+ a.handleCondition(stmt.TestExpr)for _, when := range stmt.Whens {if cw, ok := when.(*plpgsqlast.PLCaseWhen); ok {+ a.handleCondition(cw.Expr)for _, s := range cw.Body {a.walkStmt(s)}@@ -478,7 +719,7 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSet// Always collect access tables from embedded SQL for the top-level QuerySpan,// regardless of whether full analysis succeeds.- a.collectAccessTablesFromSQL(substituted)+ accesses := a.collectAccessTablesFromSQL(substituted)stmts, err := ParsePg(substituted)if err != nil {@@ -493,17 +734,16 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSetreturn nil}- query, err := a.extractor.cat.AnalyzeSelectStmt(selStmt)+ query, err := a.extractor.analyzeSelect(selStmt)if err != nil {// When full analysis fails (e.g., type mismatches with aggregate functions),// fall back to extracting column references from the parse tree and resolving// them against known tables in the catalog.+ a.extractor.collectFallbackPredicateColumns(selStmt, accesses)return a.fallbackExtractLineage(sql, selStmt)}- // Also collect from the analyzed query for precise column-level tracking- // (WHERE clause → predicate columns).- a.collectQueryPredicateColumns(query)+ a.extractor.collectStatementPredicateColumns(query, selStmt, accesses)results := a.extractor.extractFuncLineage(query)@@ -586,11 +826,8 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedExpr(expr string) base.SourceColumnSet}}if len(merged) == 0 {- varName := strings.TrimSpace(expr)- if sources, ok := a.scope.get(varName); ok {- for k, v := range sources {- merged[k] = v- }+ if stmts, err := ParsePg("SELECT " + expr); err == nil && len(stmts) == 1 {+ mergeSourceColumnSetInto(merged, a.scopeSources(stmts[0].AST))}}return merged@@ -600,7 +837,7 @@ func (a *plpgsqlAnalyzer) substituteVariables(sql string) string {stmts, err := ParsePg(sql)if err == nil && len(stmts) > 0 {if selStmt, ok := stmts[0].AST.(*ast.SelectStmt); ok {- if _, err := a.extractor.cat.AnalyzeSelectStmt(selStmt); err == nil {+ if _, err := a.extractor.analyzeSelect(selStmt); err == nil {return sql}}@@ -659,74 +896,36 @@ func extractBaseVarName(target string) string {return target}-// collectAccessTablesFromSQL extracts all table/column references from embedded SQL-// and adds them to the extractor's funcSourceColumns. This works even when-// AnalyzeSelectStmt fails, because it uses the AST-level ExtractAccessTables.-func (a *plpgsqlAnalyzer) collectAccessTablesFromSQL(sql string) {+// collectAccessTablesFromSQL extracts all table/column references from embedded SQL,+// adds them to the extractor's funcSourceColumns, and returns them. This works even+// when AnalyzeSelectStmt fails, because it uses the AST-level ExtractAccessTables.+func (a *plpgsqlAnalyzer) collectAccessTablesFromSQL(sql string) base.SourceColumnSet {+ accesses := a.extractor.accessTablesOf(sql)+ for resource := range accesses {+ a.extractor.funcSourceColumns[resource] = true+ }+ return accesses+}++// accessTablesOf returns the tables sql reads, or none when it cannot tell.+func (e *omniQuerySpanExtractor) accessTablesOf(sql string) base.SourceColumnSet {+ accesses := make(base.SourceColumnSet)accessTables, err := ExtractAccessTables(sql, ExtractAccessTablesOption{- DefaultDatabase: a.extractor.defaultDatabase,- DefaultSchema: a.extractor.searchPath[0],- SearchPath: a.extractor.searchPath,- GetDatabaseMetadata: a.extractor.gCtx.GetDatabaseMetadataFunc,- Ctx: a.extractor.ctx,- InstanceID: a.extractor.gCtx.InstanceID,+ DefaultDatabase: e.defaultDatabase,+ DefaultSchema: e.searchPath[0],+ SearchPath: e.searchPath,+ GetDatabaseMetadata: e.gCtx.GetDatabaseMetadataFunc,+ Ctx: e.ctx,+ InstanceID: e.gCtx.InstanceID,SkipMetadataValidation: false,})if err != nil {- return+ return accesses}for _, resource := range accessTables {- a.extractor.funcSourceColumns[resource] = true- }-}--// collectQueryPredicateColumns walks WHERE/HAVING clauses in an analyzed query-// and collects predicate columns (column-level) into funcPredicateColumns.-// Note: funcSourceColumns only stores table-level access (via collectAccessTablesFromSQL),-// so we don't add column-level entries to it here.-func (a *plpgsqlAnalyzer) collectQueryPredicateColumns(q *catalog.Query) {- a.collectQueryPredicateColumnsWithVisited(q, make(map[*catalog.Query]bool))-}--func (a *plpgsqlAnalyzer) collectQueryPredicateColumnsWithVisited(q *catalog.Query, visited map[*catalog.Query]bool) {- if q == nil {- return- }- if visited[q] {- return- }- visited[q] = true-- // Handle set operations recursively.- if q.SetOp != catalog.SetOpNone {- a.collectQueryPredicateColumnsWithVisited(q.LArg, visited)- a.collectQueryPredicateColumnsWithVisited(q.RArg, visited)- return- }-- // Collect from WHERE clause → predicate columns only.- if q.JoinTree != nil && q.JoinTree.Quals != nil {- a.extractor.walkExpr(q, q.JoinTree.Quals, a.extractor.funcPredicateColumns)- }-- // Collect from HAVING clause → predicate columns only.- if q.HavingQual != nil {- a.extractor.walkExpr(q, q.HavingQual, a.extractor.funcPredicateColumns)- }-- // Recurse into CTEs.- for _, cte := range q.CTEList {- if cte.Query != nil {- a.collectQueryPredicateColumnsWithVisited(cte.Query, visited)- }- }-- // Recurse into subqueries in FROM clause.- for _, rte := range q.RangeTable {- if rte.Kind == catalog.RTESubquery && rte.Subquery != nil {- a.collectQueryPredicateColumnsWithVisited(rte.Subquery, visited)- }+ accesses[resource] = true}+ return accesses}func (a *plpgsqlAnalyzer) mergeReturnResults(nCols int) []base.SourceColumnSet {@@ -796,7 +995,7 @@ func (a *plpgsqlAnalyzer) fallbackExtractLineage(origSQL string, selStmt *ast.SewhereColSet := make(base.SourceColumnSet)a.extractColumnRefsFromExpr(selStmt.WhereClause, fromTables, whereColSet)for k, v := range whereColSet {- a.extractor.funcPredicateColumns[k] = v+ a.extractor.predicateColumns[k] = v}}@@ -958,7 +1157,7 @@ func (a *plpgsqlAnalyzer) extractColumnRefsFromExpr(node ast.Node, fromTables mafilterColSet := make(base.SourceColumnSet)a.extractColumnRefsFromExpr(v.AggFilter, fromTables, filterColSet)for k, val := range filterColSet {- a.extractor.funcPredicateColumns[k] = val+ a.extractor.predicateColumns[k] = val}}case *ast.A_Expr:@@ -989,7 +1188,7 @@ func (a *plpgsqlAnalyzer) extractColumnRefsFromExpr(node ast.Node, fromTables mawhereColSet := make(base.SourceColumnSet)a.extractColumnRefsFromExpr(subSel.WhereClause, subTables, whereColSet)for k, val := range whereColSet {- a.extractor.funcPredicateColumns[k] = val+ a.extractor.predicateColumns[k] = val}}}@@ -1211,7 +1410,7 @@ func (a *plpgsqlAnalyzer) resolveThroughCTE(cteSel *ast.SelectStmt, colName striwhereColSet := make(base.SourceColumnSet)a.extractColumnRefsFromExpr(cteSel.WhereClause, cteTables, whereColSet)for k, v := range whereColSet {- a.extractor.funcPredicateColumns[k] = v+ a.extractor.predicateColumns[k] = v}}returndiff --git a/backend/plugin/parser/pg/query_span_predicate.go b/backend/plugin/parser/pg/query_span_predicate.gonew file mode 100644index 0000000000..cbb176fb9c--- /dev/null+++ b/backend/plugin/parser/pg/query_span_predicate.go@@ -0,0 +1,651 @@+package pg++import (+ "strings"++ "github.com/bytebase/omni/pg/ast"+ "github.com/bytebase/omni/pg/catalog"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// collectPredicateColumns records the columns that decide which rows q and+// every query nested in it return: those read by WHERE, JOIN ON, HAVING, and+// aggregate FILTER conditions, LIMIT and OFFSET, and the arguments of+// set-returning functions. The database evaluates conditions on unmasked values, so these columns+// decide which rows come back even when their values are masked.+func (e *omniQuerySpanExtractor) collectPredicateColumns(q *catalog.Query) {+ e.collectPredicateColumnsInStack([]*catalog.Query{q}, make(map[*catalog.Query]bool))+}++// collectStatementPredicateColumns records the predicate columns of sel,+// analyzed as query, that reads accesses.+func (e *omniQuerySpanExtractor) collectStatementPredicateColumns(query *catalog.Query, sel *ast.SelectStmt, accesses base.SourceColumnSet) {+ e.collectPredicateColumns(query)+ // The analyzer leaves a set operation's LIMIT and OFFSET out of its tree,+ // so they are resolved by name.+ r := newPredicateNameResolver(sel, e.namedFunctionReturnsSet)+ limited := false+ ast.Inspect(sel, func(n ast.Node) bool {+ if v, ok := n.(*ast.SelectStmt); ok && v.Op != ast.SETOP_NONE {+ for _, limit := range []ast.Node{v.LimitCount, v.LimitOffset} {+ if limit != nil {+ r.addReferences(limit)+ e.analyzeFunctionsCalledIn(limit)+ limited = true+ }+ }+ }+ return true+ })+ if limited {+ e.recordPredicateNames(r, accesses)+ }+}++// collectDiscardedOutputColumns records the columns a SELECT whose rows are+// discarded reads in its output. It still evaluates the output, so those+// columns decide how long it takes. query is nil when sel failed analysis.+func (e *omniQuerySpanExtractor) collectDiscardedOutputColumns(query *catalog.Query, sel *ast.SelectStmt, accesses base.SourceColumnSet) {+ if query != nil {+ for _, set := range e.extractFuncLineage(query) {+ mergeSourceColumnSetInto(e.predicateColumns, set)+ }+ return+ }+ if sel.TargetList == nil {+ return+ }+ r := newPredicateNameResolver(sel, e.namedFunctionReturnsSet)+ r.addReferences(sel.TargetList)+ e.recordPredicateNames(r, accesses)+}++func (e *omniQuerySpanExtractor) collectPredicateColumnsInStack(queryStack []*catalog.Query, visited map[*catalog.Query]bool) {+ q := currentQuery(queryStack)+ if q == nil || visited[q] {+ return+ }+ visited[q] = true++ for _, cte := range q.CTEList {+ if cte != nil {+ e.collectPredicateColumnsInStack(appendQuery(queryStack, cte.Query), visited)+ }+ }+ e.collectConditionColumns(queryStack, q.LimitCount, visited)+ e.collectConditionColumns(queryStack, q.LimitOffset, visited)+ if q.SetOp != catalog.SetOpNone {+ // INTERSECT and EXCEPT keep a row by whether the other branch has it,+ // so their branches' output columns decide which rows come back.+ filters := q.SetOp == catalog.SetOpIntersect || q.SetOp == catalog.SetOpIntersectAll ||+ q.SetOp == catalog.SetOpExcept || q.SetOp == catalog.SetOpExceptAll+ for _, branch := range []*catalog.Query{q.LArg, q.RArg} {+ if filters && branch != nil {+ for _, te := range branch.TargetList {+ if te != nil && !te.ResJunk {+ e.walkExprWithVisited(appendQuery(queryStack, branch), te.Expr, e.predicateColumns, make(map[queryExprKey]bool))+ }+ }+ }+ e.collectPredicateColumnsInStack(appendQuery(queryStack, branch), visited)+ }+ return+ }+ for _, rte := range q.RangeTable {+ if rte == nil {+ continue+ }+ if rte.Kind == catalog.RTESubquery {+ e.collectPredicateColumnsInStack(appendQuery(queryStack, rte.Subquery), visited)+ }+ for _, expr := range rte.FuncExprs {+ // A function in FROM returns the rows its arguments and the+ // conditions in its body ask for.+ call, ok := expr.(*catalog.FuncCallExpr)+ if !ok {+ e.collectConditionColumns(queryStack, expr, visited)+ continue+ }+ for _, arg := range call.Args {+ e.collectConditionColumns(queryStack, arg, visited)+ }+ for _, proc := range e.userProcsCalled(call) {+ e.analyzeFunctionBody(proc)+ }+ }+ }+ if q.JoinTree != nil {+ e.collectConditionColumns(queryStack, q.JoinTree.Quals, visited)+ for _, node := range q.JoinTree.FromList {+ e.collectJoinConditionColumns(queryStack, node, visited)+ }+ }+ e.collectConditionColumns(queryStack, q.HavingQual, visited)+ for _, te := range q.TargetList {+ if te != nil {+ e.collectNestedPredicateColumns(queryStack, te.Expr, visited)+ }+ }+}++func (e *omniQuerySpanExtractor) collectJoinConditionColumns(queryStack []*catalog.Query, node catalog.JoinNode, visited map[*catalog.Query]bool) {+ join, ok := node.(*catalog.JoinExprNode)+ if !ok || join == nil {+ return+ }+ // The analyzer turns USING and NATURAL joins into equality quals.+ e.collectConditionColumns(queryStack, join.Quals, visited)+ e.collectJoinConditionColumns(queryStack, join.Left, visited)+ e.collectJoinConditionColumns(queryStack, join.Right, visited)+}++// collectConditionColumns records every column condition reads, including the+// output columns of its subqueries, and the conditions nested in it.+func (e *omniQuerySpanExtractor) collectConditionColumns(queryStack []*catalog.Query, condition catalog.AnalyzedExpr, visited map[*catalog.Query]bool) {+ if condition == nil {+ return+ }+ e.walkExprWithVisited(queryStack, condition, e.predicateColumns, make(map[queryExprKey]bool))+ e.collectNestedPredicateColumns(queryStack, condition, visited)+}++// collectNestedPredicateColumns records the conditions nested in expr: the+// FILTER of an aggregate or window function, the arguments of a set-returning+// function, which decide how many rows it returns, and the conditions of its+// subqueries.+func (e *omniQuerySpanExtractor) collectNestedPredicateColumns(queryStack []*catalog.Query, expr catalog.AnalyzedExpr, visited map[*catalog.Query]bool) {+ catalog.WalkAnalyzedExpr(expr, func(n catalog.AnalyzedExpr) bool {+ var inputs []catalog.AnalyzedExpr+ switch v := n.(type) {+ case *catalog.SubLinkExpr:+ if v.SubQuery != nil {+ e.collectPredicateColumnsInStack(appendQuery(queryStack, v.SubQuery), visited)+ }+ case *catalog.AggExpr:+ inputs = []catalog.AnalyzedExpr{v.AggFilter}+ case *catalog.WindowFuncExpr:+ inputs = []catalog.AnalyzedExpr{v.AggFilter}+ case *catalog.FuncCallExpr:+ if e.returnsSet(v.FuncOID) {+ inputs = v.Args+ }+ default:+ }+ for _, input := range inputs {+ if input != nil {+ e.walkExprWithVisited(queryStack, input, e.predicateColumns, make(map[queryExprKey]bool))+ }+ }+ return true+ })+}++// analyzeFunctionsCalledIn analyzes the body of every user function node calls,+// which merges the body's predicates, as walkExpr does for an analyzed call.+// Without argument types the overload is unknown, so every one of the name is+// analyzed.+func (e *omniQuerySpanExtractor) analyzeFunctionsCalledIn(node ast.Node) {+ ast.Inspect(node, func(n ast.Node) bool {+ if call, ok := n.(*ast.FuncCall); ok {+ for _, proc := range e.userProcsNamed(astStrings(call.Funcname)) {+ e.analyzeFunctionBody(proc)+ }+ }+ return true+ })+}++// userProcsCalled returns the user function an analyzed call invokes. An+// opaque call, which the analyzer could not match to a function (as for a+// stand-in), may invoke any user function of its name.+func (e *omniQuerySpanExtractor) userProcsCalled(call *catalog.FuncCallExpr) []*catalog.UserProc {+ if proc := e.cat.GetUserProcByOID(call.FuncOID); proc != nil {+ return []*catalog.UserProc{proc}+ }+ if call.FuncOID == 0 {+ return e.userProcsNamed([]string{call.FuncName})+ }+ return nil+}++// userProcsNamed returns every user function, overloads included, that a+// possibly schema-qualified name may call.+func (e *omniQuerySpanExtractor) userProcsNamed(names []string) []*catalog.UserProc {+ if len(names) == 0 {+ return nil+ }+ name := names[len(names)-1]+ schemas := e.searchPath+ if len(names) > 1 {+ schemas = []string{names[len(names)-2]}+ }+ var procs []*catalog.UserProc+ for _, schema := range schemas {+ for _, row := range e.cat.QueryPgProc(schema) {+ if strings.EqualFold(row.ProName, name) {+ if proc := e.cat.GetUserProcByOID(row.OID); proc != nil {+ procs = append(procs, proc)+ }+ }+ }+ }+ return procs+}++// namedFunctionReturnsSet reports whether a function of that name is known to+// return a set.+func (e *omniQuerySpanExtractor) namedFunctionReturnsSet(name string) bool {+ for _, proc := range e.userProcsNamed([]string{name}) {+ if proc.RetSet {+ return true+ }+ }+ for _, proc := range e.cat.LookupProcByName(strings.ToLower(name)) {+ if proc.RetSet {+ return true+ }+ }+ return false+}++// returnsSet reports whether the function is known to return a set. An+// unknown function, analyzed opaquely, is not.+func (e *omniQuerySpanExtractor) returnsSet(oid uint32) bool {+ if proc := e.cat.GetUserProcByOID(oid); proc != nil {+ return proc.RetSet+ }+ if proc := e.cat.LookupProcByOID(oid); proc != nil {+ return proc.RetSet+ }+ return false+}++// collectFallbackPredicateColumns stands in for collectPredicateColumns when+// analysis fails. Without scopes it over-approximates by name, so that an+// analysis failure never lets a condition on a masked column through.+func (e *omniQuerySpanExtractor) collectFallbackPredicateColumns(stmt ast.Node, accesses base.SourceColumnSet) {+ e.analyzeFunctionsCalledIn(stmt)+ r := newPredicateNameResolver(stmt, e.namedFunctionReturnsSet)+ for _, name := range r.usingNames {+ r.addName(name)+ }+ for _, condition := range r.conditions {+ r.addReferences(condition)+ }+ // The joined columns are unknown without scopes, so a NATURAL JOIN reads+ // every column of both sides.+ for _, join := range r.naturalJoins {+ r.addJoinedRelation(join.Larg)+ r.addJoinedRelation(join.Rarg)+ }+ for _, setOperation := range r.filteringSetOperations {+ r.addSetOperationOutputs(setOperation)+ }+ e.recordPredicateNames(r, accesses)+}++func (e *omniQuerySpanExtractor) recordPredicateNames(r *predicateNameResolver, accesses base.SourceColumnSet) {+ mergeSourceColumnSetInto(e.predicateColumns, e.columnsNamed(r, accesses))+}++// columnsNamed returns the columns of accesses that r's names match.+func (e *omniQuerySpanExtractor) columnsNamed(r *predicateNameResolver, accesses base.SourceColumnSet) base.SourceColumnSet {+ columns := make(base.SourceColumnSet)+ for resource := range accesses {+ rel := e.cat.GetRelation(resource.Schema, resource.Table)+ if rel == nil {+ continue+ }+ table := strings.ToLower(resource.Table)+ for _, column := range rel.Columns {+ name := strings.ToLower(column.Name)+ if r.wholeRows[table] || r.anyTable[name] || r.ofTable[table][name] {+ columns[base.ColumnResource{+ Database: resource.Database,+ Schema: resource.Schema,+ Table: resource.Table,+ Column: column.Name,+ }] = true+ }+ }+ }+ return columns+}++// returningLineage returns, for each column a DML statement's RETURNING list+// outputs, the columns of accesses it reads, matched by name. It returns nil+// for a statement without RETURNING.+func (e *omniQuerySpanExtractor) returningLineage(stmt ast.Node, accesses base.SourceColumnSet) []base.SourceColumnSet {+ var returning *ast.List+ var target *ast.RangeVar+ switch v := stmt.(type) {+ case *ast.InsertStmt:+ returning, target = v.ReturningList, v.Relation+ case *ast.UpdateStmt:+ returning, target = v.ReturningList, v.Relation+ case *ast.DeleteStmt:+ returning, target = v.ReturningList, v.Relation+ default:+ }+ if returning == nil {+ return nil+ }+ var lineage []base.SourceColumnSet+ for _, item := range returning.Items {+ rt, ok := item.(*ast.ResTarget)+ if !ok {+ continue+ }+ if isStarTarget(rt) && target != nil {+ // RETURNING * outputs every column of the target table in order.+ for resource := range accesses {+ if !strings.EqualFold(resource.Table, target.Relname) {+ continue+ }+ if rel := e.cat.GetRelation(resource.Schema, resource.Table); rel != nil {+ for _, column := range rel.Columns {+ lineage = append(lineage, base.SourceColumnSet{{+ Database: resource.Database,+ Schema: resource.Schema,+ Table: resource.Table,+ Column: column.Name,+ }: true})+ }+ }+ break+ }+ continue+ }+ r := newPredicateNameResolver(stmt, e.namedFunctionReturnsSet)+ r.addReferences(rt.Val)+ lineage = append(lineage, e.columnsNamed(r, accesses))+ }+ return lineage+}++// predicateNameResolver resolves the column references of conditions by name.+// A reference qualified by a table or its alias matches that table's column;+// any other reference matches the same-named column of every table, and an+// output name of a subquery or CTE also reads the columns that produce it.+type predicateNameResolver struct {+ // The WHERE (of SELECT, UPDATE, and DELETE), JOIN ON, HAVING, and FILTER+ // conditions, LIMIT and OFFSET,+ // arguments of FROM and set-returning functions, USING column names,+ // NATURAL joins, and INTERSECT and EXCEPT operations of the statement, at+ // any depth.+ conditions []ast.Node+ usingNames []string+ naturalJoins []*ast.JoinExpr+ filteringSetOperations []*ast.SelectStmt+ // projections maps an output name to the expressions that produce it. A+ // subquery or CTE name maps to its whole query, so a whole-row reference+ // to it reads every column.+ projections map[string][]ast.Node+ // relations maps a name or alias in FROM to the relations it may name; an+ // alias reused across scopes names several.+ relations map[string]map[string]bool+ // derived holds the names of subqueries and CTEs.+ derived map[string]bool++ anyTable map[string]bool+ ofTable map[string]map[string]bool+ wholeRows map[string]bool+}++func newPredicateNameResolver(stmt ast.Node, returnsSet func(name string) bool) *predicateNameResolver {+ r := &predicateNameResolver{+ projections: make(map[string][]ast.Node),+ relations: make(map[string]map[string]bool),+ derived: make(map[string]bool),+ anyTable: make(map[string]bool),+ ofTable: make(map[string]map[string]bool),+ wholeRows: make(map[string]bool),+ }+ project := func(name string, nodes ...ast.Node) {+ if name != "" {+ r.projections[strings.ToLower(name)] = append(r.projections[strings.ToLower(name)], nodes...)+ }+ }+ ast.Inspect(stmt, func(n ast.Node) bool {+ switch v := n.(type) {+ case *ast.SelectStmt:+ r.conditions = append(r.conditions, v.WhereClause, v.HavingClause, v.LimitCount, v.LimitOffset)+ if v.Op == ast.SETOP_INTERSECT || v.Op == ast.SETOP_EXCEPT {+ r.filteringSetOperations = append(r.filteringSetOperations, v)+ }+ if v.TargetList != nil {+ for _, item := range v.TargetList.Items {+ if rt, ok := item.(*ast.ResTarget); ok {+ name := rt.Name+ if name == "" {+ name = figureResTargetName(rt)+ }+ project(name, rt.Val)+ }+ }+ }+ case *ast.UpdateStmt:+ r.conditions = append(r.conditions, v.WhereClause)+ case *ast.DeleteStmt:+ r.conditions = append(r.conditions, v.WhereClause)+ case *ast.JoinExpr:+ r.conditions = append(r.conditions, v.Quals)+ r.usingNames = append(r.usingNames, astStrings(v.UsingClause)...)+ if v.IsNatural {+ r.naturalJoins = append(r.naturalJoins, v)+ }+ case *ast.FuncCall:+ r.conditions = append(r.conditions, v.AggFilter)+ if names := astStrings(v.Funcname); len(names) > 0 && returnsSet(names[len(names)-1]) {+ r.conditions = append(r.conditions, v.Args)+ }+ case *ast.RangeFunction:+ // A function in FROM returns the rows its arguments ask for.+ r.conditions = append(r.conditions, v.Functions)+ case *ast.CommonTableExpr:+ r.derived[strings.ToLower(v.Ctename)] = true+ project(v.Ctename, v.Ctequery)+ for i, name := range astStrings(v.Aliascolnames) {+ project(name, outputsAt(v.Ctequery, i)...)+ }+ case *ast.RangeSubselect:+ if v.Alias != nil {+ r.derived[strings.ToLower(v.Alias.Aliasname)] = true+ project(v.Alias.Aliasname, v.Subquery)+ for i, name := range astStrings(v.Alias.Colnames) {+ project(name, outputsAt(v.Subquery, i)...)+ }+ }+ case *ast.RangeVar:+ relation := strings.ToLower(v.Relname)+ r.addRelationName(relation, relation)+ if v.Alias != nil {+ r.addRelationName(v.Alias.Aliasname, relation)+ }+ default:+ }+ return true+ })+ return r+}++func (r *predicateNameResolver) addRelationName(name, relation string) {+ name = strings.ToLower(name)+ if name == "" {+ return+ }+ if r.relations[name] == nil {+ r.relations[name] = make(map[string]bool)+ }+ r.relations[name][relation] = true+}++// relationsNamed returns the tables name may refer to, and the subqueries or+// CTEs it may refer to instead.+func (r *predicateNameResolver) relationsNamed(name string) (map[string]bool, []string) {+ name = strings.ToLower(name)+ tables := make(map[string]bool)+ var derived []string+ if r.derived[name] {+ derived = append(derived, name)+ }+ for relation := range r.relations[name] {+ if r.derived[relation] {+ derived = append(derived, relation)+ } else {+ tables[relation] = true+ }+ }+ return tables, derived+}++// addSetOperationOutputs adds the outputs of every branch of sel, which an+// INTERSECT or EXCEPT compares.+func (r *predicateNameResolver) addSetOperationOutputs(sel *ast.SelectStmt) {+ if sel == nil {+ return+ }+ if sel.Op != ast.SETOP_NONE {+ r.addSetOperationOutputs(sel.Larg)+ r.addSetOperationOutputs(sel.Rarg)+ return+ }+ if sel.TargetList == nil {+ return+ }+ for _, item := range sel.TargetList.Items {+ rt, ok := item.(*ast.ResTarget)+ if !ok {+ continue+ }+ if !isStarTarget(rt) {+ r.addReferences(rt.Val)+ continue+ }+ if sel.FromClause != nil {+ for _, from := range sel.FromClause.Items {+ r.addJoinedRelation(from)+ }+ }+ }+}++func (r *predicateNameResolver) addJoinedRelation(node ast.Node) {+ switch v := node.(type) {+ case *ast.RangeVar:+ if v.Alias != nil && v.Alias.Aliasname != "" {+ r.addWholeRow(v.Alias.Aliasname)+ } else {+ r.addWholeRow(v.Relname)+ }+ case *ast.RangeSubselect:+ r.addReferences(v.Subquery)+ case *ast.JoinExpr:+ r.addJoinedRelation(v.Larg)+ r.addJoinedRelation(v.Rarg)+ default:+ }+}++func (r *predicateNameResolver) addReferences(node ast.Node) {+ ast.Inspect(node, func(n ast.Node) bool {+ if ref, ok := n.(*ast.ColumnRef); ok {+ r.addReference(ref)+ }+ return true+ })+}++func (r *predicateNameResolver) addReference(ref *ast.ColumnRef) {+ if ref.Fields == nil {+ return+ }+ var fields []string+ star := false+ for _, field := range ref.Fields.Items {+ switch v := field.(type) {+ case *ast.String:+ fields = append(fields, v.Str)+ case *ast.A_Star:+ star = true+ default:+ }+ }+ if len(fields) == 0 {+ return+ }+ last := fields[len(fields)-1]+ // t.* reads every column of t; a bare t may be a column or a whole-row+ // reference.+ if star || len(fields) == 1 {+ r.addWholeRow(last)+ return+ }+ tables, derived := r.relationsNamed(fields[len(fields)-2])+ for table := range tables {+ if r.ofTable[table] == nil {+ r.ofTable[table] = make(map[string]bool)+ }+ r.ofTable[table][strings.ToLower(last)] = true+ }+ if len(derived) > 0 || len(tables) == 0 {+ r.addName(last)+ }+}++func (r *predicateNameResolver) addWholeRow(name string) {+ tables, derived := r.relationsNamed(name)+ for table := range tables {+ r.wholeRows[table] = true+ }+ for _, relation := range derived {+ r.addName(relation)+ }+ r.addName(name)+}++func (r *predicateNameResolver) addName(name string) {+ name = strings.ToLower(name)+ if name == "" || r.anyTable[name] {+ return+ }+ r.anyTable[name] = true+ for _, projection := range r.projections[name] {+ r.addReferences(projection)+ }+}++// outputsAt returns the expressions that produce the i-th output column of+// query, or query itself when they cannot be told apart.+func outputsAt(query ast.Node, i int) []ast.Node {+ sel, ok := query.(*ast.SelectStmt)+ if !ok || sel == nil {+ return []ast.Node{query}+ }+ if sel.Op != ast.SETOP_NONE {+ return append(outputsAt(sel.Larg, i), outputsAt(sel.Rarg, i)...)+ }+ if sel.TargetList == nil || i >= len(sel.TargetList.Items) {+ return []ast.Node{query}+ }+ for _, item := range sel.TargetList.Items[:i+1] {+ if rt, ok := item.(*ast.ResTarget); !ok || isStarTarget(rt) {+ return []ast.Node{query}+ }+ }+ return []ast.Node{sel.TargetList.Items[i].(*ast.ResTarget).Val}+}++func astStrings(list *ast.List) []string {+ if list == nil {+ return nil+ }+ var result []string+ for _, item := range list.Items {+ if s, ok := item.(*ast.String); ok {+ result = append(result, s.Str)+ }+ }+ return result+}diff --git a/backend/plugin/parser/pg/query_span_predicate_test.go b/backend/plugin/parser/pg/query_span_predicate_test.gonew file mode 100644index 0000000000..00e659b2e6--- /dev/null+++ b/backend/plugin/parser/pg/query_span_predicate_test.go@@ -0,0 +1,294 @@+package pg++import (+ "context"+ "slices"+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/bytebase/omni/pg/ast"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func predicateColumnNames(columns base.SourceColumnSet) []string {+ names := []string{}+ for column := range columns {+ if column.Column != "" {+ names = append(names, column.Table+"."+column.Column)+ }+ }+ slices.Sort(names)+ return names+}++func TestGetQuerySpanPredicateColumns(t *testing.T) {+ testCases := []struct {+ name string+ statement string+ want []string+ }{+ {"where", "SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},+ {"where function", "SELECT id FROM t WHERE ascii(substr(ssn, 1, 1)) > 70", []string{"t.ssn"}},+ {"row comparison", "SELECT id FROM t WHERE (ssn, id) > ('1', 0)", []string{"t.id", "t.ssn"}},+ {"join on", "SELECT o.amt FROM o JOIN t ON t.id = o.id AND t.ssn = '1'", []string{"o.id", "t.id", "t.ssn"}},+ {"left join on", "SELECT o.amt FROM o LEFT JOIN t ON t.id = o.id AND t.ssn = '1'", []string{"o.id", "t.id", "t.ssn"}},+ {"join using", "SELECT a.id FROM t a JOIN t b USING (ssn)", []string{"t.ssn"}},+ {"having", "SELECT id FROM t GROUP BY id HAVING max(ssn) > '1'", []string{"t.ssn"}},+ {"in subquery", "SELECT amt FROM o WHERE id IN (SELECT id FROM t WHERE ssn = '1')", []string{"o.id", "t.id", "t.ssn"}},+ {"subquery output", "SELECT amt FROM o WHERE '1' = (SELECT max(ssn) FROM t)", []string{"t.ssn"}},+ {"correlated exists", "SELECT amt FROM o WHERE EXISTS (SELECT 1 FROM t WHERE t.id = o.id AND t.ssn = '1')", []string{"o.id", "t.id", "t.ssn"}},+ {"subquery in select list", "SELECT (SELECT count(*) FROM t WHERE ssn = '1') AS c", []string{"t.ssn"}},+ {"derived table", "SELECT id FROM (SELECT * FROM t WHERE ssn = '1') s", []string{"t.ssn"}},+ {"derived table output", "SELECT s.id FROM (SELECT id, ssn AS x FROM t) s WHERE s.x = '1'", []string{"t.ssn"}},+ {"lateral subquery", "SELECT o.amt FROM o, LATERAL (SELECT 1 FROM t WHERE t.id = o.id AND t.ssn = '1') l", []string{"o.id", "t.id", "t.ssn"}},+ {"cte", "WITH c AS (SELECT id FROM t WHERE ssn = '1') SELECT id FROM c", []string{"t.ssn"}},+ {"cte output", "WITH c AS (SELECT id, ssn FROM t) SELECT id FROM c WHERE ssn = '1'", []string{"t.ssn"}},+ {"set operation arm", "SELECT id FROM t WHERE ssn = '1' UNION SELECT id FROM o", []string{"t.ssn"}},+ {"except filters by the other branch", "SELECT 'guess' AS x EXCEPT SELECT ssn FROM t", []string{"t.ssn"}},+ {"intersect filters by the other branch", "SELECT 'guess' AS x INTERSECT ALL SELECT ssn FROM t", []string{"t.ssn"}},+ {"union filters nothing", "SELECT 'guess' AS x UNION SELECT ssn FROM t", []string{}},+ {"explain analyze", "EXPLAIN ANALYZE SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},+ {"aggregate filter", "SELECT sum(id) FILTER (WHERE ssn = '1') AS x FROM t", []string{"t.ssn"}},+ {"natural join", "SELECT 1 FROM t NATURAL JOIN (SELECT '1'::text AS ssn) x", []string{"t.ssn"}},+ {"full join using", "SELECT id FROM t FULL JOIN o USING (id) WHERE id > 0", []string{"o.id", "t.id"}},+ {"limit subquery", "SELECT id FROM t LIMIT (SELECT count(*) FROM t WHERE ssn = '1')", []string{"t.ssn"}},+ {"limit value", "SELECT id FROM t LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},+ {"set operation limit", "SELECT 1 AS x UNION ALL SELECT 2 LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},+ {"set-returning function in from", "SELECT count(*) AS c FROM t CROSS JOIN LATERAL generate_series(1, ascii(substr(t.ssn, 1, 1)))", []string{"t.ssn"}},+ {"set-returning function in select list", "SELECT generate_series(1, ascii(substr(ssn, 1, 1))) AS g FROM t", []string{"t.ssn"}},+ {"scalar function in select list", "SELECT upper(ssn) AS u FROM t", []string{}},+ {"unknown function", "SELECT id FROM t WHERE levenshtein(ssn, 'a') > 0", []string{"t.ssn"}},+ {"unknown function in from", "SELECT t.id FROM t, LATERAL unknown_rows(t.ssn) f WHERE f.a = 1", []string{"t.ssn"}},+ {"select list only", "SELECT ssn FROM t", []string{}},+ {"condition in select list", "SELECT CASE WHEN ssn = '1' THEN 1 END AS x FROM t", []string{}},+ }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ span := spanFor(t, tc.statement, healthySchema())+ require.Equal(t, tc.want, predicateColumnNames(span.PredicateColumns))+ })+ }+}++func TestGetQuerySpanPredicateColumnsOfFunctionBodies(t *testing.T) {+ schema := healthySchema()+ // * over an unknown function fails analysis of the body that uses it.+ unanalyzable := "(SELECT * FROM o CROSS JOIN unknown_rows() u) o JOIN t ON t.id = o.id AND t.ssn = guess"+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{+ {Name: "probe", Definition: "CREATE FUNCTION public.probe(guess text) RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT t.id FROM t WHERE t.ssn = guess; END; $$"},+ {Name: "probe_sql", Definition: "CREATE FUNCTION public.probe_sql(guess text) RETURNS SETOF int LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = guess $$"},+ {Name: "probe_fixed", Definition: "CREATE FUNCTION public.probe_fixed() RETURNS SETOF int LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = 'guess' $$"},+ {Name: "probe_multi", Definition: "CREATE FUNCTION public.probe_multi() RETURNS int LANGUAGE sql AS $$ SELECT pg_sleep(0) FROM t WHERE ssn = 'guess'; SELECT 1 $$"},+ {Name: "probe_perform", Definition: "CREATE FUNCTION public.probe_perform() RETURNS int LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(0) FROM t WHERE ssn = 'guess'; RETURN 1; END; $$"},+ {Name: "probe_if", Definition: "CREATE FUNCTION public.probe_if() RETURNS int LANGUAGE plpgsql AS $$ BEGIN IF (SELECT ssn FROM t LIMIT 1) = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},+ {Name: "probe_variable", Definition: "CREATE FUNCTION public.probe_variable() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t WHERE id = 1; IF v = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},+ {Name: "probe_update", Definition: "CREATE FUNCTION public.probe_update() RETURNS int LANGUAGE sql AS $$ UPDATE t SET id = id WHERE ssn = 'guess'; SELECT 1 $$"},+ {Name: "probe_exec", Definition: "CREATE FUNCTION public.probe_exec() RETURNS int LANGUAGE plpgsql AS $$ BEGIN DELETE FROM t WHERE ssn = 'guess'; RETURN 1; END; $$"},+ {Name: "probe_table", Definition: "CREATE FUNCTION public.probe_table() RETURNS TABLE(id int) LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = 'guess' $$"},+ {Name: "probe_record", Definition: "CREATE FUNCTION public.probe_record() RETURNS int LANGUAGE plpgsql AS $$ DECLARE r record; BEGIN SELECT ssn INTO r FROM t LIMIT 1; IF r.ssn = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},+ {Name: "probe_execute", Definition: "CREATE FUNCTION public.probe_execute() RETURNS int LANGUAGE plpgsql AS $$ BEGIN EXECUTE 'SELECT pg_sleep(0) FROM t ' || 'WHERE ssn = ''guess'''; RETURN 1; END; $$"},+ {Name: "probe_count", Definition: "CREATE FUNCTION public.probe_count() RETURNS int LANGUAGE sql AS $$ SELECT count(*)::int FROM t WHERE ssn = 'guess' $$"},+ {Name: "probe_bool", Definition: "CREATE FUNCTION public.probe_bool() RETURNS boolean LANGUAGE sql AS $$ SELECT EXISTS (SELECT 1 FROM t WHERE ssn = 'guess') $$"},+ {Name: "probe_raise", Definition: "CREATE FUNCTION public.probe_raise() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t LIMIT 1; RAISE EXCEPTION '%', v; END; $$"},+ // The overload that reads ssn loads first, so a body chosen by name alone+ // would be the harmless one.+ {Name: "probe_over", Definition: "CREATE FUNCTION public.probe_over(x text) RETURNS boolean LANGUAGE sql AS $$ SELECT EXISTS (SELECT 1 FROM t WHERE ssn = x) $$"},+ {Name: "probe_over", Definition: "CREATE FUNCTION public.probe_over(x int) RETURNS boolean LANGUAGE sql AS $$ SELECT true $$"},+ // A missing return type makes the catalog install these as stand-ins.+ {Name: "probe_stand_in", Definition: "CREATE FUNCTION public.probe_stand_in(x text) RETURNS SETOF missing_tbl LANGUAGE sql AS $$ SELECT * FROM t WHERE ssn = x $$"},+ {Name: "probe_stand_in", Definition: "CREATE FUNCTION public.probe_stand_in(x int) RETURNS SETOF missing_tbl LANGUAGE sql AS $$ SELECT * FROM o $$"},+ {Name: "probe_stand_in_pl", Definition: "CREATE FUNCTION public.probe_stand_in_pl(x text) RETURNS SETOF missing_tbl LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT * FROM t WHERE ssn = x; END; $$"},+ {Name: "probe_discard", Definition: "CREATE FUNCTION public.probe_discard() RETURNS int LANGUAGE sql AS $$ SELECT pg_sleep(CASE WHEN ssn = 'guess' THEN 0 ELSE 0 END) FROM t; SELECT 1 $$"},+ {Name: "probe_declare", Definition: "CREATE FUNCTION public.probe_declare() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text := (SELECT ssn FROM t LIMIT 1); BEGIN RAISE EXCEPTION '%', v; END; $$"},+ {Name: "proc_check", Definition: "CREATE PROCEDURE public.proc_check(x text) LANGUAGE plpgsql AS $$ BEGIN IF x = 'guess' THEN RAISE EXCEPTION 'hit'; END IF; END; $$"},+ {Name: "probe_call_subquery", Definition: "CREATE FUNCTION public.probe_call_subquery() RETURNS int LANGUAGE plpgsql AS $$ BEGIN CALL proc_check((SELECT ssn FROM t LIMIT 1)); RETURN 1; END; $$"},+ {Name: "probe_call", Definition: "CREATE FUNCTION public.probe_call() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t LIMIT 1; CALL proc_check(v); RETURN 1; END; $$"},+ // The last SELECT reads a table the snapshot lacks, so the body fails analysis.+ {Name: "probe_failing", Definition: "CREATE FUNCTION public.probe_failing() RETURNS SETOF record LANGUAGE sql AS $$ SELECT pg_sleep(0) FROM t WHERE ssn = 'guess'; SELECT * FROM missing_tbl $$"},+ {Name: "probe_for", Definition: "CREATE FUNCTION public.probe_for() RETURNS SETOF int LANGUAGE plpgsql AS $$ BEGIN FOR i IN 1..(SELECT ascii(ssn) FROM t LIMIT 1) LOOP RETURN NEXT i; END LOOP; END; $$"},+ {Name: "probe_join", Definition: "CREATE FUNCTION public.probe_join(guess text) RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT o.id FROM " + unanalyzable + "; END; $$"},+ {Name: "probe_join_sql", Definition: "CREATE FUNCTION public.probe_join_sql(guess text) RETURNS SETOF int LANGUAGE sql AS $$ SELECT o.id FROM " + unanalyzable + " $$"},+ }+ testCases := []struct {+ statement string+ want []string+ }{+ {"SELECT count(*) AS c FROM probe('guess')", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_sql('guess')", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_fixed()", []string{"t.ssn"}},+ {"SELECT probe_multi() AS x", []string{"t.ssn"}},+ {"SELECT probe_perform() AS x", []string{"t.ssn"}},+ {"SELECT probe_if() AS x", []string{"t.ssn"}},+ {"SELECT probe_variable() AS x", []string{"t.id", "t.ssn"}},+ {"SELECT count(*) AS c FROM probe_for()", []string{"t.ssn"}},+ {"SELECT probe_update() AS x", []string{"t.ssn"}},+ {"SELECT probe_record() AS x", []string{"t.ssn"}},+ {"SELECT probe_raise() AS x", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_stand_in('guess')", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_stand_in_pl('guess')", []string{"t.ssn"}},+ {"SELECT probe_discard() AS x", []string{"t.ssn"}},+ {"SELECT probe_declare() AS x", []string{"t.ssn"}},+ {"SELECT probe_call() AS x", []string{"t.ssn"}},+ {"SELECT probe_call_subquery() AS x", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_failing() AS f(a int)", []string{"t.ssn"}},+ {"SELECT probe_over('guess') AS b", []string{"t.ssn"}},+ {"SELECT probe_over(1) AS b", []string{}},+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE probe_over(1)", []string{"t.ssn"}},+ // omni leaves a set operation's LIMIT out of its tree.+ {"SELECT 1 AS x UNION ALL SELECT 2 LIMIT probe_count()", []string{"t.ssn"}},+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE probe_bool()", []string{"t.ssn"}},+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE public.probe_bool()", []string{"t.ssn"}},+ {"SELECT probe_execute() AS x", []string{"t.ssn"}},+ {"SELECT probe_exec() AS x", []string{"t.ssn"}},+ // * over an unknown function sends the outer query to the fallback.+ {"SELECT * FROM probe_table() p CROSS JOIN unknown_rows() u", []string{"t.ssn"}},+ {"SELECT count(*) AS c FROM probe_join('guess')", []string{"o.id", "t.id", "t.ssn"}},+ {"SELECT count(*) AS c FROM probe_join_sql('guess')", []string{"o.id", "t.id", "t.ssn"}},+ }+ for _, tc := range testCases {+ t.Run(tc.statement, func(t *testing.T) {+ span := spanFor(t, tc.statement, schema)+ require.Equal(t, tc.want, predicateColumnNames(span.PredicateColumns))+ })+ }+}++func TestGetQuerySpanResultLineage(t *testing.T) {+ testCases := []struct {+ name string+ statement string+ want []string+ }{+ {"set operation in subquery", "SELECT s.a FROM (SELECT ssn AS a FROM t UNION SELECT email FROM t) s", []string{"t.email", "t.ssn"}},+ {"set operation in cte", "WITH c AS (SELECT ssn FROM t UNION SELECT email FROM t) SELECT ssn FROM c", []string{"t.email", "t.ssn"}},+ {"except returns left branch rows", "SELECT s.a FROM (SELECT ssn AS a FROM t EXCEPT SELECT email FROM t) s", []string{"t.ssn"}},+ {"window before column in subquery", "SELECT s.x FROM (SELECT row_number() OVER (ORDER BY ssn) AS rn, email AS x FROM t) s", []string{"t.email"}},+ {"full join using column", "SELECT id FROM t FULL JOIN o USING (id)", []string{"o.id", "t.id"}},+ {"aggregate order by", "SELECT string_agg(email, ',' ORDER BY ssn) AS x FROM t", []string{"t.email", "t.ssn"}},+ {"unknown function in from", "SELECT f.a FROM t CROSS JOIN LATERAL unknown_rows(t.ssn) f", []string{"t.ssn"}},+ }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ span := spanFor(t, tc.statement, healthySchema())+ require.Len(t, span.Results, 1)+ require.Equal(t, tc.want, predicateColumnNames(span.Results[0].SourceColumns))+ })+ }+}++func TestGetQuerySpanResultLineageOfFunctionBodies(t *testing.T) {+ schema := healthySchema()+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{+ {Name: "read_cursor", Definition: "CREATE FUNCTION public.read_cursor() RETURNS text LANGUAGE plpgsql AS $$ DECLARE c CURSOR FOR SELECT ssn FROM t; v text; BEGIN OPEN c; FETCH c INTO v; RETURN v; END; $$"},+ {Name: "loop_cursor", Definition: "CREATE FUNCTION public.loop_cursor() RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE c CURSOR FOR SELECT ssn FROM t; BEGIN FOR r IN c LOOP RETURN NEXT r.ssn; END LOOP; END; $$"},+ {Name: "returning_sql", Definition: "CREATE FUNCTION public.returning_sql() RETURNS SETOF text LANGUAGE sql AS $$ UPDATE t SET id = id RETURNING ssn $$"},+ {Name: "returning_table", Definition: "CREATE FUNCTION public.returning_table() RETURNS TABLE(i int, e text, s text) LANGUAGE sql AS $$ DELETE FROM t WHERE false RETURNING * $$"},+ {Name: "returning_pl", Definition: "CREATE FUNCTION public.returning_pl() RETURNS text LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN UPDATE t SET id = id WHERE id = 1 RETURNING ssn INTO v; RETURN v; END; $$"},+ }+ testCases := []struct {+ statement string+ want []string+ }{+ {"SELECT read_cursor() AS x", []string{"t.ssn"}},+ {"SELECT loop_cursor() AS x", []string{"t.ssn"}},+ {"SELECT returning_sql() AS x", []string{"t.ssn"}},+ {"SELECT returning_pl() AS x", []string{"t.ssn"}},+ {"SELECT r.s FROM returning_table() r", []string{"t.ssn"}},+ }+ for _, tc := range testCases {+ t.Run(tc.statement, func(t *testing.T) {+ span := spanFor(t, tc.statement, schema)+ require.Len(t, span.Results, 1)+ require.Equal(t, tc.want, predicateColumnNames(span.Results[0].SourceColumns))+ })+ }+}++func TestGetQuerySpanKeepsSQLBodyAccesses(t *testing.T) {+ schema := degradedSchema()+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{+ {Name: "probe_sql", Definition: "CREATE FUNCTION public.probe_sql() RETURNS int LANGUAGE sql AS $$ SELECT 1 FROM t WHERE secret = 'guess' $$"},+ }+ span := spanFor(t, "SELECT probe_sql() AS x", schema)+ require.Contains(t, span.SourceColumns, base.ColumnResource{Database: "db", Schema: "public", Table: "t"})+ require.NotNil(t, span.UnresolvedColumnsError, "t, read only in the body's condition, was synced with no columns")+}++func TestGetQuerySpanFallbackKeepsFunctionAccesses(t *testing.T) {+ schema := degradedSchema()+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{+ {Name: "probe_t", Definition: "CREATE FUNCTION public.probe_t() RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT 1 FROM t WHERE secret = 'guess'; END; $$"},+ }+ // * over an unknown function sends the statement to the fallback.+ span := spanFor(t, "SELECT * FROM probe_t() p CROSS JOIN unknown_rows() u", schema)+ require.Contains(t, span.SourceColumns, base.ColumnResource{Database: "db", Schema: "public", Table: "t"})+ require.NotNil(t, span.UnresolvedColumnsError, "t, read only inside the function, was synced with no columns")+}++func newFallbackTestExtractor(t *testing.T) *omniQuerySpanExtractor {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{healthySchema()})+ e := newOmniQuerySpanExtractor("db", nil, base.GetQuerySpanContext{GetDatabaseMetadataFunc: getter, ListDatabaseNamesFunc: lister})+ e.ctx = context.Background()+ require.NoError(t, e.initCatalog())+ return e+}++func TestCollectFallbackPredicateColumns(t *testing.T) {+ testCases := []struct {+ name string+ statement string+ want []string+ }{+ {"where", "SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},+ {"qualified reference matches its table", "SELECT o.amt FROM o JOIN t ON o.id = t.id WHERE o.amt > 0", []string{"o.amt", "o.id", "t.id"}},+ {"unqualified reference matches every table", "SELECT 1 FROM o JOIN t USING (id) WHERE id > 0", []string{"o.id", "t.id"}},+ {"reused alias matches every table it names", "SELECT 1 FROM t x WHERE EXISTS (SELECT 1 FROM o x WHERE x.id = 1)", []string{"o.id", "t.id"}},+ {"subquery in a condition reads every name in it", "SELECT 1 FROM o WHERE EXISTS (WITH c AS (SELECT id FROM t WHERE ssn = '1') SELECT 1 FROM c)", []string{"o.id", "t.id", "t.ssn"}},+ {"cte column alias", "WITH c(a, b) AS (SELECT id, ssn FROM t) SELECT a FROM c WHERE b = '1'", []string{"t.ssn"}},+ {"recursive cte column alias", "WITH RECURSIVE c(a, n) AS (SELECT ssn, 1 FROM t UNION ALL SELECT a, n + 1 FROM c WHERE n < 3) SELECT a FROM c", []string{}},+ {"derived column alias", "SELECT 1 FROM (SELECT id, ssn FROM t) s(a, b) WHERE s.b = '1'", []string{"t.ssn"}},+ {"output name", "SELECT 1 FROM (SELECT ssn AS x FROM t) s WHERE x = '1'", []string{"t.ssn"}},+ {"whole row", "SELECT 1 FROM t WHERE t::text LIKE '%1%'", []string{"t.email", "t.id", "t.ssn"}},+ {"aliased whole row", "SELECT 1 FROM t x WHERE ROW(x.*) IS NOT NULL", []string{"t.email", "t.id", "t.ssn"}},+ {"join using", "SELECT 1 FROM t a JOIN t b USING (ssn)", []string{"t.ssn"}},+ {"having", "SELECT id FROM t GROUP BY id HAVING max(ssn) > '1'", []string{"t.ssn"}},+ {"aggregate filter", "SELECT count(*) FILTER (WHERE ssn = '1') FROM t", []string{"t.ssn"}},+ {"natural join reads both sides", "SELECT 1 FROM o NATURAL JOIN (SELECT id FROM t) x", []string{"o.amt", "o.id", "t.id"}},+ {"select list only", "SELECT ssn FROM t", []string{}},+ {"except filters by the other branch", "SELECT 'guess' EXCEPT SELECT ssn FROM t", []string{"t.ssn"}},+ {"intersect over star", "SELECT 1, 'a', 'b' INTERSECT SELECT * FROM t", []string{"t.email", "t.id", "t.ssn"}},+ {"union filters nothing", "SELECT 'guess' UNION SELECT ssn FROM t", []string{}},+ {"limit value", "SELECT * FROM t LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},+ {"function in from", "SELECT * FROM t CROSS JOIN LATERAL generate_series(1, ascii(substr(t.ssn, 1, 1))) g", []string{"t.ssn"}},+ {"set-returning function in select list", "SELECT generate_series(1, ascii(substr(ssn, 1, 1))) FROM t", []string{"t.ssn"}},+ {"scalar function in select list", "SELECT upper(ssn) FROM t", []string{}},+ }+ accesses := base.SourceColumnSet{+ {Database: "db", Schema: "public", Table: "t"}: true,+ {Database: "db", Schema: "public", Table: "o"}: true,+ }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ e := newFallbackTestExtractor(t)+ stmts, err := ParsePg(tc.statement)+ require.NoError(t, err)+ e.collectFallbackPredicateColumns(stmts[0].AST, accesses)+ require.Equal(t, tc.want, predicateColumnNames(e.predicateColumns))+ })+ }+}++func TestGetQuerySpanFallbackKeepsPredicateColumns(t *testing.T) {+ // * over the unknown columns of a function fails omni's analysis.+ statement := "SELECT * FROM o JOIN t ON t.id = o.id AND t.ssn = '1' CROSS JOIN unknown_rows() f"+ stmts, err := ParsePg(statement)+ require.NoError(t, err)+ _, err = newFallbackTestExtractor(t).analyzeSelect(stmts[0].AST.(*ast.SelectStmt))+ require.Error(t, err, "the statement must fail analysis to reach the fallback")++ span := spanFor(t, statement, healthySchema())+ require.Equal(t, []string{"o.id", "t.id", "t.ssn"}, predicateColumnNames(span.PredicateColumns))+}diff --git a/backend/plugin/parser/pg/query_span_test.go b/backend/plugin/parser/pg/query_span_test.goindex cdfde7d715..945078f859 100644--- a/backend/plugin/parser/pg/query_span_test.go+++ b/backend/plugin/parser/pg/query_span_test.go@@ -632,16 +632,12 @@ func TestIsUltimatelyPlainColumnStopsOnCyclicSubquery(t *testing.T) {})}-func TestCollectQueryPredicateColumnsStopsOnCyclicQuery(t *testing.T) {+func TestCollectPredicateColumnsStopsOnCyclicQuery(t *testing.T) {mustNotOverflow(t, func() {q := &catalog.Query{}q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})- analyzer := &plpgsqlAnalyzer{- extractor: extractor,- scope: newVariableScope(nil),- }- analyzer.collectQueryPredicateColumns(q)+ extractor.collectPredicateColumns(q)})}diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.goindex 28adc2e193..d76b25483b 100644--- a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go@@ -90,6 +90,7 @@ func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {"SELECT email FROM public.t","SELECT t.email FROM public.t","SELECT * FROM public.t WHERE email = 'x'",+ "EXPLAIN ANALYZE SELECT * FROM public.t WHERE email = 'x'","SELECT id, email FROM public.t ORDER BY ssn","WITH c AS (SELECT * FROM public.t) SELECT * FROM c",// A non-recursive CTE cannot reference itself; the inner t is the physical table.diff --git a/backend/plugin/parser/pg/test-data/query_span.yaml b/backend/plugin/parser/pg/test-data/query_span.yamlindex dcec60021e..62a7209f74 100644--- a/backend/plugin/parser/pg/test-data/query_span.yaml+++ b/backend/plugin/parser/pg/test-data/query_span.yaml@@ -855,6 +855,21 @@table: task_runcolumn: ""predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: issue+ column: id+ - server: ""+ database: db+ schema: public+ table: issue+ column: payload+ - server: ""+ database: db+ schema: public+ table: issue+ column: pipeline_id- server: ""database: dbschema: public@@ -865,11 +880,41 @@schema: publictable: principalcolumn: email+ - server: ""+ database: db+ schema: public+ table: principal+ column: id+ - server: ""+ database: db+ schema: public+ table: task+ column: id+ - server: ""+ database: db+ schema: public+ table: task+ column: pipeline_id+ - server: ""+ database: db+ schema: public+ table: task+ column: status- server: ""database: dbschema: publictable: task_runcolumn: creator_id+ - server: ""+ database: db+ schema: public+ table: task_run+ column: status+ - server: ""+ database: db+ schema: public+ table: task_run+ column: task_id- description: Test For PLpgSQL User Defined Function in expressionstatement: SELECT plpgsql_t(1);defaultDatabase: db@@ -1114,7 +1159,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: to_jsonb accept table as argument with from table aliasstatement: SELECT to_jsonb(t1) FROM t t1;defaultDatabase: db@@ -1390,7 +1440,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Test For Complex PLpgSQL Function As Table Sourcestatement: SELECT * FROM plpgsql_complex_t(1);defaultDatabase: db@@ -1458,7 +1513,17 @@schema: publictable: t2column: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a+ - server: ""+ database: db+ schema: public+ table: t2+ column: a- description: Test For PLpgSQL Function As Table Sourcestatement: SELECT * FROM plpgsql_t(1);defaultDatabase: db@@ -1507,7 +1572,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Test For Cascade Function As Table Sourcestatement: SELECT * FROM cascade_t(1);defaultDatabase: db@@ -1560,7 +1630,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Test For Function As Table Sourcestatement: SELECT * FROM select_t(1);defaultDatabase: db@@ -1609,7 +1684,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Test For Explain statementsstatement: EXPLAIN SELECT 1;defaultDatabase: db@@ -2060,7 +2140,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Correlated subquery in the target field clause.statement: SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total_paintings FROM galleries g;defaultDatabase: db@@ -2478,7 +2563,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: Simple CTEstatement: WITH t1 AS (SELECT * FROM t) SELECT * FROM t1;defaultDatabase: db@@ -3026,7 +3116,17 @@schema: publictable: userscolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: payments+ column: user_id+ - server: ""+ database: db+ schema: public+ table: users+ column: id- description: jsonb_array_elements as table sourcestatement: SELECT * FROM t, jsonb_array_elements(a) AS elems WHERE id = 1;defaultDatabase: db@@ -3079,7 +3179,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: jsonb_array_elements with column aliasstatement: SELECT * FROM t, jsonb_array_elements(a) AS elems(val) WHERE id = 1;defaultDatabase: db@@ -3132,7 +3237,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: jsonb_array_elements_text as table source with aliasstatement: SELECT * FROM t CROSS JOIN LATERAL jsonb_array_elements_text(a) AS part WHERE id = 1;defaultDatabase: db@@ -3185,7 +3295,12 @@schema: publictable: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: public+ table: t+ column: a- description: array subscript in select liststatement: |-SELECTdiff --git a/backend/tests/sensitive_data_test.go b/backend/tests/sensitive_data_test.goindex aa3c9577d4..e2714d7067 100644--- a/backend/tests/sensitive_data_test.go+++ b/backend/tests/sensitive_data_test.go@@ -6,12 +6,15 @@ import ("fmt""strings""testing"+ "time""connectrpc.com/connect""github.com/google/go-cmp/cmp""github.com/stretchr/testify/require"+ "google.golang.org/genproto/googleapis/type/expr""google.golang.org/protobuf/testing/protocmp""google.golang.org/protobuf/types/known/durationpb"+ "google.golang.org/protobuf/types/known/fieldmaskpb"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")@@ -92,6 +95,7 @@ func TestSensitiveData(t *testing.T) {name varchar(220),author varchar(220));+ CREATE TABLE ghost();`insertData = `INSERT INTO tech_book VALUES@@ -285,7 +289,7 @@ func TestSensitiveData(t *testing.T) {// but the masking pipeline should redact it.errorLeakResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{Name: database.Name,- Statement: "SELECT author::uuid FROM tech_book WHERE id = 1",+ Statement: "SELECT author::uuid FROM tech_book WHERE name = 'bytebase'",}))a.NoError(err)a.Equal(1, len(errorLeakResp.Msg.Results))@@ -297,7 +301,7 @@ func TestSensitiveData(t *testing.T) {// Same query on a non-masked column should preserve the original error.nonMaskedErrorResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{Name: database.Name,- Statement: "SELECT name::uuid FROM tech_book WHERE id = 1",+ Statement: "SELECT name::uuid FROM tech_book WHERE name = 'bytebase'",}))a.NoError(err)a.Equal(1, len(nonMaskedErrorResp.Msg.Results))@@ -305,6 +309,64 @@ func TestSensitiveData(t *testing.T) {// The error should NOT be redacted — name is not masked.a.NotContains(nonMaskedErrorResp.Msg.Results[0].Error, "data masking policies")+ const refusedPredicate = "Using sensitive columns in query conditions is not allowed"+ for _, statement := range []string{+ "SELECT name FROM tech_book WHERE author = 'bber'",+ "SELECT b.name FROM tech_book b JOIN tech_book a ON a.id = b.id AND ascii(substr(a.author, 1, 1)) > 70",+ "SELECT name FROM tech_book WHERE EXISTS (SELECT 1 FROM tech_book t WHERE t.author LIKE 'E%')",+ "EXPLAIN ANALYZE SELECT name FROM tech_book WHERE author LIKE 'E%'",+ "SELECT count(*) FILTER (WHERE author LIKE 'E%') FROM tech_book",+ "SELECT count(*) FROM tech_book NATURAL JOIN (SELECT 'bber'::varchar AS author) x",+ // Running this would fail on the cast, so the refusal proves it never ran.+ "SELECT name FROM tech_book WHERE author::int = 1",+ } {+ resp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: statement,+ }))+ a.NoError(err, statement)+ a.Len(resp.Msg.Results, 1, statement)+ a.Contains(resp.Msg.Results[0].Error, refusedPredicate, statement)+ a.Empty(resp.Msg.Results[0].Rows, statement)+ }++ unmaskedPredicateResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: "SELECT name FROM tech_book WHERE name = 'bytebase'",+ }))+ a.NoError(err)+ a.Len(unmaskedPredicateResp.Msg.Results, 1)+ a.Empty(unmaskedPredicateResp.Msg.Results[0].Error)+ a.Len(unmaskedPredicateResp.Msg.Results[0].Rows, 1)++ for _, statement := range []string{+ "SELECT a FROM (SELECT author AS a FROM tech_book UNION ALL SELECT name FROM tech_book) s",+ "WITH c AS (SELECT author FROM tech_book UNION SELECT name FROM tech_book) SELECT author FROM c",+ "SELECT s.x FROM (SELECT row_number() OVER (ORDER BY name) AS rn, author AS x FROM tech_book) s",+ } {+ resp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: statement,+ }))+ a.NoError(err, statement)+ a.Len(resp.Msg.Results, 1, statement)+ a.Empty(resp.Msg.Results[0].Error, statement)+ a.NotEmpty(resp.Msg.Results[0].Rows, statement)+ for _, row := range resp.Msg.Results[0].Rows {+ a.NotContains([]string{"bber", "Egor Rogov", "Martin Kleppmann"}, row.Values[0].GetStringValue(), statement)+ }+ }++ batchResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: "SELECT name FROM tech_book; SELECT name FROM tech_book WHERE author = 'bber';",+ }))+ a.NoError(err)+ a.Len(batchResp.Msg.Results, 2)+ a.Empty(batchResp.Msg.Results[0].Error)+ a.Len(batchResp.Msg.Results[0].Rows, 3)+ a.Contains(batchResp.Msg.Results[1].Error, refusedPredicate)+// Created directly on the target, after the last schema sync, so the synced// schema does not know the function or how many columns it returns.targetDB, err := sql.Open("pgx", fmt.Sprintf("postgresql://postgres:root-password@%s:%s/%s?sslmode=disable", pgContainer.GetHost(), pgContainer.GetPort(), databaseName))@@ -332,4 +394,40 @@ func TestSensitiveData(t *testing.T) {a.Len(unmaskedResp.Msg.Results, 1)a.Equal([]string{"a", "b"}, unmaskedResp.Msg.Results[0].ColumnNames)a.Len(unmaskedResp.Msg.Results[0].Rows, 1)++ // ghost synced with no columns. A column added outside Bytebase is masked by+ // name once it syncs, so the condition on it must be refused before the+ // query runs: a run would take the 30 seconds pg_sleep asks for. The query+ // syncs the database, so it comes after the unsynced_rows checks.+ _, err = targetDB.ExecContext(ctx, "ALTER TABLE ghost ADD COLUMN ghost_secret text; INSERT INTO ghost VALUES ('x')")+ a.NoError(err)+ workspace, err := ctl.workspaceServiceClient.GetWorkspace(ctx, connect.NewRequest(&v1pb.GetWorkspaceRequest{Name: "workspaces/-"}))+ a.NoError(err)+ _, err = ctl.orgPolicyServiceClient.UpdatePolicy(ctx, connect.NewRequest(&v1pb.UpdatePolicyRequest{+ Policy: &v1pb.Policy{+ Name: workspace.Msg.Name + "/policies/masking_rule",+ Type: v1pb.PolicyType_MASKING_RULE,+ Policy: &v1pb.Policy_MaskingRulePolicy{+ MaskingRulePolicy: &v1pb.MaskingRulePolicy{+ Rules: []*v1pb.MaskingRulePolicy_MaskingRule{{+ Id: "6b1e7c2a-3f4d-4e5a-9b8c-0d1e2f3a4b5c",+ Condition: &expr.Expr{Expression: `resource.column_name == "ghost_secret"`},+ SemanticType: "default",+ }},+ },+ },+ },+ AllowMissing: true,+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"masking_rule_policy"}},+ }))+ a.NoError(err)+ start := time.Now()+ ghostResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{+ Name: database.Name,+ Statement: "SELECT 1 FROM ghost WHERE ghost_secret = 'x' AND pg_sleep(30) IS NOT NULL",+ }))+ a.NoError(err)+ a.Less(time.Since(start), 20*time.Second)+ a.Len(ghostResp.Msg.Results, 1)+ a.Contains(ghostResp.Msg.Results[0].Error, refusedPredicate)}diff --git a/go.mod b/go.modindex f5eca782da..9825ef498c 100644--- a/go.mod+++ b/go.mod@@ -36,7 +36,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e+ github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861github.com/caarlos0/env/v11 v11.4.1github.com/cenkalti/backoff/v7 v7.0.0github.com/cockroachdb/cockroach-go/v2 v2.4.3diff --git a/go.sum b/go.sumindex 9158695c9f..01e9438af9 100644--- a/go.sum+++ b/go.sum@@ -252,8 +252,8 @@ github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735 h1:v8y6mXeHmYVsdMgithub.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735/go.mod h1:vTHjcvfn8Pmf65WjnV1ytTCYyC7xNhFjrniv4YWvtA8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e h1:HliM6J1o0GjVkQ7j6MarLVhqy0ezGqVrZohVelbxDHU=-github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=+github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861 h1:IV5+STWCkVNLz6E82uqSeFIZg56uUiVaGpg4ET2kYLk=+github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=