Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(masking)!: refuse PostgreSQL queries whose conditions read masked columns (#21651)

rebelice799732c12b1e
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.go
index 27672dc6f1..4bb66187ff 100644
--- a/backend/api/v1/predicate_columns_check.go
+++ b/backend/api/v1/predicate_columns_check.go
@@ -2,7 +2,10 @@ package v1
import (
"context"
+ "slices"
+ "strings"
+ "connectrpc.com/connect"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
@@ -11,6 +14,32 @@ import (
"github.com/bytebase/bytebase/backend/store"
)
+// refuseSensitivePredicateColumns refuses, before execution, statements whose
+// conditions read a column masked for user. The database evaluates conditions
+// on unmasked values, so the rows it returns, their count, and the time it
+// takes would reveal them.
+func refuseSensitivePredicateColumns(ctx context.Context, stores *store.Store, spans []*base.QuerySpan, instance *store.InstanceMessage, user *store.UserMessage) error {
+ if !isPredicateColumnsCheckEnabled(instance.Metadata.GetEngine()) {
+ return nil
+ }
+ sensitiveColumns, err := NewQueryResultMasker(stores).ExtractSensitivePredicateColumns(ctx, spans, instance, user)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, errors.New(err.Error()))
+ }
+ for _, columns := range sensitiveColumns {
+ if len(columns) == 0 {
+ continue
+ }
+ names := make([]string, 0, len(columns))
+ for _, column := range columns {
+ names = append(names, column.String())
+ }
+ slices.Sort(names)
+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("Using sensitive columns in query conditions is not allowed: %s", strings.Join(names, ", ")))
+ }
+ return nil
+}
+
func (s *QueryResultMasker) ExtractSensitivePredicateColumns(ctx context.Context, spans []*base.QuerySpan, instance *store.InstanceMessage, user *store.UserMessage) ([][]base.ColumnResource, error) {
var result [][]base.ColumnResource
@@ -118,5 +147,5 @@ func (m *maskingLevelEvaluator) isSensitiveColumn(database *store.DatabaseMessag
}
func isPredicateColumnsCheckEnabled(engine storepb.Engine) bool {
- return engine == storepb.Engine_MSSQL
+ return engine == storepb.Engine_MSSQL || engine == storepb.Engine_POSTGRES
}
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 34e69fdace..f9d0175537 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -602,14 +602,21 @@ func replaceBackupTableWithSource(ctx context.Context, stores *store.Store, inst
}
for _, span := range spans {
- span.SourceColumns = generateNewSourceColumnSet(instance.Metadata.GetEngine(), span.SourceColumns, schema)
- for _, result := range span.Results {
- result.SourceColumns = generateNewSourceColumnSet(instance.Metadata.GetEngine(), result.SourceColumns, schema)
- }
+ remapBackupTableColumns(instance.Metadata.GetEngine(), span, schema)
}
return nil
}
+// remapBackupTableColumns points the columns span reads from backup tables at
+// the source columns, whose masking policies apply to the backups too.
+func remapBackupTableColumns(engine storepb.Engine, span *parserbase.QuerySpan, schema *model.SchemaMetadata) {
+ span.SourceColumns = generateNewSourceColumnSet(engine, span.SourceColumns, schema)
+ span.PredicateColumns = generateNewSourceColumnSet(engine, span.PredicateColumns, schema)
+ for i := range span.Results {
+ span.Results[i].SourceColumns = generateNewSourceColumnSet(engine, span.Results[i].SourceColumns, schema)
+ }
+}
+
func generateNewSourceColumnSet(engine storepb.Engine, origin parserbase.SourceColumnSet, schema *model.SchemaMetadata) parserbase.SourceColumnSet {
result := make(parserbase.SourceColumnSet)
for column := range origin {
@@ -833,31 +840,12 @@ func queryRetry(
schemaSyncer *schemasync.Syncer,
multiStatement bool,
) ([]*v1pb.QueryResult, []*parserbase.QuerySpan, time.Duration, error) {
- var sensitivePredicateColumns [][]parserbase.ColumnResource
// An explain request carries the bare statements the driver will wrap, so
// their spans name the databases the plan describes.
- spans, err := parserbase.GetQuerySpan(
- ctx,
- parserbase.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),
- ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),
- GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),
- },
- instance.Metadata.GetEngine(),
- statements,
- database.DatabaseName,
- queryContext.Schema,
- !store.IsObjectCaseSensitive(instance),
- )
+ spans, err := buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema)
if err != nil {
return nil, nil, time.Duration(0), err
}
- // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.
- // If err != nil, this function will return the original spans.
- if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {
- slog.Debug("failed to replace backup table with source", log.BBError(err))
- }
if optionalAccessCheck != nil {
// A smuggled write (EXPLAIN ANALYZE of a write) is refused earlier, in
// the Query handler, before it reaches execution.
@@ -866,20 +854,26 @@ func queryRetry(
}
slog.Debug("optional access check", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))
}
- if !queryContext.Explain && !queryContext.SkipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {
- masker := NewQueryResultMasker(stores)
- sensitivePredicateColumns, err = masker.ExtractSensitivePredicateColumns(ctx, spans, instance, user)
- if err != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.New(err.Error()))
- }
- slog.Debug("extract sensitive predicate columns", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))
- }
-
maskingEnabled := !queryContext.Explain && !queryContext.SkipMasking &&
licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil
queryContext.MaskingEnabled = maskingEnabled
if maskingEnabled {
+ // Columns a stale snapshot lacks can hide a condition from the predicate
+ // check, so sync before execution. MaskResults refuses the result if the
+ // columns stay unresolved. Permanently empty tables re-sync on every query
+ // until BYT-10074 adds throttling.
+ if unresolved := unresolvedColumnDatabases(spans, instance); len(unresolved) > 0 {
+ if err := syncDatabaseMetadata(ctx, stores, schemaSyncer, instance, unresolved); err != nil {
+ return nil, nil, time.Duration(0), err
+ }
+ if spans, err = buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema); err != nil {
+ return nil, nil, time.Duration(0), err
+ }
+ }
+ if err := refuseSensitivePredicateColumns(ctx, stores, spans, instance, user); err != nil {
+ return nil, nil, time.Duration(0), err
+ }
if err := preExecuteMaskingCheck(ctx, stores, instance.Metadata.GetEngine(), database, spans); err != nil {
return nil, nil, time.Duration(0), err
}
@@ -913,21 +907,7 @@ func queryRetry(
syncDatabaseMap := make(map[string]bool)
for i, r := range results {
- if i >= len(spans) {
- continue
- }
- // Re-sync even for error results: MaskResults also refuses partial rows.
- // Permanently empty tables re-sync on every query until BYT-10074 adds throttling.
- if maskingEnabled && maskingBlockedByUnresolvedColumns(spans[i], instance) {
- for _, dbName := range spans[i].UnresolvedColumnsError.Databases() {
- slog.Debug("database metadata need to sync: unresolved columns",
- slog.String("instance", instance.ResourceID),
- slog.String("database", dbName),
- slog.String("detail", spans[i].UnresolvedColumnsError.Error()))
- syncDatabaseMap[dbName] = true
- }
- }
- if r.Error != "" {
+ if i >= len(spans) || r.Error != "" {
continue
}
if spans[i].NotFoundError != nil {
@@ -938,55 +918,22 @@ func queryRetry(
}
}
- // Sync database metadata.
- for accessDatabaseName := range syncDatabaseMap {
- slog.Debug("sync database metadata", slog.String("instance", instance.ResourceID), slog.String("database", accessDatabaseName))
- d, err := stores.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID, DatabaseName: &accessDatabaseName})
- if err != nil {
- return nil, nil, duration, err
- }
- if d == nil {
- // Leave the span error in place so masking can reject an untracked database.
- slog.Debug("skip metadata sync: database not tracked",
- slog.String("instance", instance.ResourceID),
- slog.String("database", accessDatabaseName))
- continue
- }
- if err := schemaSyncer.SyncDatabaseSchema(ctx, d); err != nil {
- return nil, nil, duration, errors.Wrapf(err, "failed to sync database schema for database %q", accessDatabaseName)
- }
+ if err := syncDatabaseMetadata(ctx, stores, schemaSyncer, instance, syncDatabaseMap); err != nil {
+ return nil, nil, duration, err
}
// Retry getting query span.
if len(syncDatabaseMap) > 0 {
slog.Debug("retry query after sync metadata", slog.String("instance", instance.ResourceID), slog.String("database", database.DatabaseName))
- spans, err = parserbase.GetQuerySpan(
- ctx,
- parserbase.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),
- ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),
- GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),
- },
- instance.Metadata.GetEngine(),
- statements,
- database.DatabaseName,
- queryContext.Schema,
- !store.IsObjectCaseSensitive(instance),
- )
+ spans, err = buildQuerySpans(ctx, stores, instance, database, statements, queryContext.Schema)
if err != nil {
return nil, nil, time.Duration(0), err
}
- // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.
- // If err != nil, this function will return the original spans.
- if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {
- slog.Debug("failed to replace backup table with source", log.BBError(err))
- }
- if !queryContext.SkipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {
- masker := NewQueryResultMasker(stores)
- sensitivePredicateColumns, err = masker.ExtractSensitivePredicateColumns(ctx, spans, instance, user)
- if err != nil {
- return nil, nil, time.Duration(0), connect.NewError(connect.CodeInternal, errors.New(err.Error()))
+ // A table the first span did not find may surface a condition the
+ // pre-execution check could not see.
+ if maskingEnabled {
+ if err := refuseSensitivePredicateColumns(ctx, stores, spans, instance, user); err != nil {
+ return nil, nil, duration, err
}
}
}
@@ -1017,33 +964,83 @@ func queryRetry(
if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {
return nil, nil, duration, connect.NewError(connect.CodeInternal, errors.New(err.Error()))
}
-
- for i, result := range results {
- if i >= len(sensitivePredicateColumns) {
- continue
- }
- if len(sensitivePredicateColumns[i]) == 0 {
- continue
- }
- result.Error = getSensitivePredicateColumnErrorMessages(sensitivePredicateColumns[i])
- result.Rows = nil
- result.RowsCount = 0
- }
}
}
return results, spans, duration, nil
}
-func getSensitivePredicateColumnErrorMessages(sensitiveColumns []parserbase.ColumnResource) string {
- var buf bytes.Buffer
- _, _ = buf.WriteString("Using sensitive columns in WHERE clause is not allowed: ")
- for j, column := range sensitiveColumns {
- if j > 0 {
- _, _ = buf.WriteString(", ")
+// buildQuerySpans analyzes statements for access checks and masking.
+func buildQuerySpans(
+ ctx context.Context,
+ stores *store.Store,
+ instance *store.InstanceMessage,
+ database *store.DatabaseMessage,
+ statements []parserbase.Statement,
+ schema string,
+) ([]*parserbase.QuerySpan, error) {
+ spans, err := parserbase.GetQuerySpan(
+ ctx,
+ parserbase.GetQuerySpanContext{
+ InstanceID: instance.ResourceID,
+ GetDatabaseMetadataFunc: parsercontext.BuildGetDatabaseMetadataFunc(stores),
+ ListDatabaseNamesFunc: parsercontext.BuildListDatabaseNamesFunc(stores),
+ GetLinkedDatabaseMetadataFunc: parsercontext.BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),
+ },
+ instance.Metadata.GetEngine(),
+ statements,
+ database.DatabaseName,
+ schema,
+ !store.IsObjectCaseSensitive(instance),
+ )
+ if err != nil {
+ return nil, err
+ }
+ // After replacing backup table with source, we can apply the original access check and mask sensitive data for backup table.
+ // If err != nil, this function will return the original spans.
+ if err := replaceBackupTableWithSource(ctx, stores, instance, database, spans); err != nil {
+ slog.Debug("failed to replace backup table with source", log.BBError(err))
+ }
+ return spans, nil
+}
+
+// unresolvedColumnDatabases returns the databases whose stale metadata keeps
+// masking from resolving a span's columns.
+func unresolvedColumnDatabases(spans []*parserbase.QuerySpan, instance *store.InstanceMessage) map[string]bool {
+ databases := make(map[string]bool)
+ for _, span := range spans {
+ if !maskingBlockedByUnresolvedColumns(span, instance) {
+ continue
+ }
+ for _, name := range span.UnresolvedColumnsError.Databases() {
+ slog.Debug("database metadata need to sync: unresolved columns",
+ slog.String("instance", instance.ResourceID),
+ slog.String("database", name),
+ slog.String("detail", span.UnresolvedColumnsError.Error()))
+ databases[name] = true
+ }
+ }
+ return databases
+}
+
+func syncDatabaseMetadata(ctx context.Context, stores *store.Store, schemaSyncer *schemasync.Syncer, instance *store.InstanceMessage, databaseNames map[string]bool) error {
+ for name := range databaseNames {
+ slog.Debug("sync database metadata", slog.String("instance", instance.ResourceID), slog.String("database", name))
+ d, err := stores.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID, DatabaseName: &name})
+ if err != nil {
+ return err
+ }
+ if d == nil {
+ // Leave the span error in place so masking can reject an untracked database.
+ slog.Debug("skip metadata sync: database not tracked",
+ slog.String("instance", instance.ResourceID),
+ slog.String("database", name))
+ continue
+ }
+ if err := schemaSyncer.SyncDatabaseSchema(ctx, d); err != nil {
+ return errors.Wrapf(err, "failed to sync database schema for database %q", name)
}
- _, _ = buf.WriteString(column.String())
}
- return buf.String()
+ return nil
}
// queryRetryStopOnError runs the query and stops on encountering errors.
diff --git a/backend/api/v1/sql_service_test.go b/backend/api/v1/sql_service_test.go
index 66fa49c32d..1f0058e5e2 100644
--- a/backend/api/v1/sql_service_test.go
+++ b/backend/api/v1/sql_service_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"connectrpc.com/connect"
+ metadatapb "github.com/bytebase/omni/metadata"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
@@ -16,6 +17,7 @@ import (
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store"
+ "github.com/bytebase/bytebase/backend/store/model"
)
func TestSQLIAMDatabaseResourceUsesCanonicalInstanceScope(t *testing.T) {
@@ -642,3 +644,27 @@ func TestExplainOfAWriteNamesItsTables(t *testing.T) {
require.NoError(t, f.check(ctx, "[email protected]", true, "CALL p()", call))
requireExplainDenied(t, f.check(ctx, "[email protected]", true, "CALL p()", call), permission.SQLExplain, "instances/test/databases/db_a")
}
+
+func TestRemapBackupTableColumns(t *testing.T) {
+ dbMeta := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: common.BackupDatabaseNameOfEngine(storepb.Engine_POSTGRES),
+ Tables: []*metadatapb.TableMetadata{{Name: "bak_t", Comment: "bytebase, source table (public, t)"}},
+ }},
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)
+ backup := parserbase.ColumnResource{Database: "db", Schema: common.BackupDatabaseNameOfEngine(storepb.Engine_POSTGRES), Table: "bak_t", Column: "secret"}
+ source := parserbase.ColumnResource{Database: "db", Schema: "public", Table: "t", Column: "secret"}
+ span := &parserbase.QuerySpan{
+ SourceColumns: parserbase.SourceColumnSet{backup: true},
+ PredicateColumns: parserbase.SourceColumnSet{backup: true},
+ Results: []parserbase.QuerySpanResult{{Name: "secret", SourceColumns: parserbase.SourceColumnSet{backup: true}}},
+ }
+
+ remapBackupTableColumns(storepb.Engine_POSTGRES, span, getSchemaMetadata(storepb.Engine_POSTGRES, dbMeta))
+
+ want := parserbase.SourceColumnSet{source: true}
+ require.Equal(t, want, span.SourceColumns)
+ require.Equal(t, want, span.PredicateColumns)
+ require.Equal(t, want, span.Results[0].SourceColumns)
+}
diff --git a/backend/plugin/parser/pg/query_span.go b/backend/plugin/parser/pg/query_span.go
index fae50bcbed..6e5e458e41 100644
--- a/backend/plugin/parser/pg/query_span.go
+++ b/backend/plugin/parser/pg/query_span.go
@@ -57,17 +57,20 @@ type omniQuerySpanExtractor struct {
metaCache map[string]*model.DatabaseMetadata
cat *catalog.Catalog
funcBodyCache map[uint32]*funcBodyAnalysis
- // funcOrigDefs stores the original (non-stubbed) function definitions keyed
- // by lowercase function name. Used by analyzeFunctionBody to get the real
- // body when it was stubbed during catalog loading.
- funcOrigDefs map[string]string
+ // snapshotFuncs holds the snapshot's definition of every overload, keyed by
+ // lowercase "schema.name". Used by analyzeFunctionBody for a function the
+ // catalog installed as a stand-in, listed in standInFuncs by the same key;
+ // the stand-in may stand for any overload of the name.
+ snapshotFuncs map[string][]snapshotFunc
+ standInFuncs map[string]bool
// funcSourceColumns accumulates table-level access (column="") discovered inside
// function bodies. Merged into the top-level QuerySpan.SourceColumns.
funcSourceColumns base.SourceColumnSet
- // funcPredicateColumns accumulates columns used in fallback WHERE/JOIN
- // conditions. Function body analysis records predicates separately so callers
- // can decide whether they enter the top-level QuerySpan.PredicateColumns.
- funcPredicateColumns base.SourceColumnSet
+ // predicateColumns accumulates the columns that decide which rows the
+ // statement returns; it becomes QuerySpan.PredicateColumns. Function body
+ // analysis collects into a set of its own, cached per function, and merges
+ // it into the caller's.
+ predicateColumns base.SourceColumnSet
// fallbackCTEMap holds CTE definitions during fallback column extraction.
// Set by extractFallbackColumns and used by extractColumnsFromRangeVar
// to resolve CTE references.
@@ -83,14 +86,15 @@ func newOmniQuerySpanExtractor(defaultDatabase string, searchPath []string, gCtx
searchPath = []string{"public"}
}
return &omniQuerySpanExtractor{
- defaultDatabase: defaultDatabase,
- searchPath: searchPath,
- gCtx: gCtx,
- metaCache: make(map[string]*model.DatabaseMetadata),
- funcBodyCache: make(map[uint32]*funcBodyAnalysis),
- funcOrigDefs: make(map[string]string),
- funcSourceColumns: make(base.SourceColumnSet),
- funcPredicateColumns: make(base.SourceColumnSet),
+ defaultDatabase: defaultDatabase,
+ searchPath: searchPath,
+ gCtx: gCtx,
+ metaCache: make(map[string]*model.DatabaseMetadata),
+ funcBodyCache: make(map[uint32]*funcBodyAnalysis),
+ snapshotFuncs: make(map[string][]snapshotFunc),
+ standInFuncs: make(map[string]bool),
+ funcSourceColumns: make(base.SourceColumnSet),
+ predicateColumns: make(base.SourceColumnSet),
}
}
@@ -127,6 +131,12 @@ func (e *omniQuerySpanExtractor) initCatalog() error {
if err != nil {
return errors.Wrapf(err, "failed to load the catalog for %q", e.defaultDatabase)
}
+ for key := range report.Degraded {
+ if name, ok := strings.CutPrefix(key, "func:"); ok {
+ name, _, _ = strings.Cut(name, "|")
+ e.standInFuncs[strings.ToLower(name)] = true
+ }
+ }
if len(report.Degraded) > 0 || len(report.Missing) > 0 {
slog.Debug("catalog loaded with stand-ins",
slog.String("database", e.defaultDatabase),
@@ -135,16 +145,15 @@ func (e *omniQuerySpanExtractor) initCatalog() error {
)
}
- // Populate original (non-stubbed) function definitions for PL/pgSQL body
- // analysis.
for _, s := range meta.GetProto().GetSchemas() {
for _, f := range s.GetFunctions() {
if f.Definition == "" {
continue
}
- origBody := extractFuncBodyFromDef(f.Definition)
- if origBody != "" {
- e.funcOrigDefs[strings.ToLower(f.Name)] = origBody
+ _, body, language := parseFuncDefForLineage(f.Definition)
+ if body != "" {
+ key := strings.ToLower(s.Name + "." + f.Name)
+ e.snapshotFuncs[key] = append(e.snapshotFuncs[key], snapshotFunc{language: language, body: body})
}
}
}
@@ -285,13 +294,27 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
queryType, isExplainAnalyze := classifyQueryType(omniStmts[0].AST, allSystems)
// Step 5: Return early for non-SELECT queries, keeping the tables the
- // statement touches and no result columns.
+ // statement touches and no result columns. EXPLAIN ANALYZE runs its query
+ // and reports the rows each condition kept, so its conditions still count
+ // as predicates.
if queryType != base.Select || isExplainAnalyze {
+ var unresolved *base.UnresolvedColumnsError
+ if isExplainAnalyze {
+ if err := e.collectExplainedPredicateColumns(omniStmts[0].AST, accessesMap); err != nil {
+ return nil, err
+ }
+ e.mergeFunctionAccesses(accessesMap)
+ // Columns a stale snapshot lacks can hide a condition, so the
+ // caller syncs before running it, as for the SELECT itself.
+ unresolved = e.unresolvedColumnsError(accessesMap)
+ }
return &base.QuerySpan{
- Type: queryType,
- ExplainAnalyze: isExplainAnalyze,
- SourceColumns: accessesMap,
- Results: []base.QuerySpanResult{},
+ Type: queryType,
+ ExplainAnalyze: isExplainAnalyze,
+ SourceColumns: accessesMap,
+ PredicateColumns: e.predicateColumns,
+ Results: []base.QuerySpanResult{},
+ UnresolvedColumnsError: unresolved,
}, nil
}
@@ -310,26 +333,24 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
return nil, errors.Wrapf(err, "failed to initialize catalog")
}
- query, err := e.cat.AnalyzeSelectStmt(selStmt)
+ query, err := e.analyzeSelect(selStmt)
if err != nil {
+ e.collectFallbackPredicateColumns(selStmt, accessesMap)
// Before falling back, try to handle user-defined table-returning functions
// that omni's AnalyzeSelectStmt can't resolve (e.g., RETURNS TABLE functions
// used as table sources: SELECT * FROM func()).
- if results := e.tryUserFuncTableSource(selStmt, accessesMap); results != nil {
- return &base.QuerySpan{
- Type: base.Select,
- SourceColumns: accessesMap,
- Results: results,
- UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
- }, nil
- }
- // Fail-open: return access tables with best-effort column names and lineage
- // when analysis fails (e.g., unsupported built-in functions).
+ results := e.tryUserFuncTableSource(selStmt, accessesMap)
+ if results == nil {
+ // Fail-open: return access tables with best-effort column names and lineage
+ // when analysis fails (e.g., unsupported built-in functions).
+ results = e.extractFallbackColumns(selStmt)
+ }
+ e.mergeFunctionAccesses(accessesMap)
return &base.QuerySpan{
Type: base.Select,
SourceColumns: accessesMap,
- Results: e.extractFallbackColumns(selStmt),
- PredicateColumns: e.funcPredicateColumns,
+ Results: results,
+ PredicateColumns: e.predicateColumns,
UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
}, nil
}
@@ -338,21 +359,63 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
// SourceColumns (table-level access) is already captured in accessesMap
// from ExtractAccessTables above — no need to re-extract from the query.
results := e.extractLineage(query, selStmt)
+ e.collectStatementPredicateColumns(query, selStmt, accessesMap)
- // Merge columns discovered inside function bodies into the top-level sets.
- for col := range e.funcSourceColumns {
- accessesMap[col] = true
- }
+ e.mergeFunctionAccesses(accessesMap)
return &base.QuerySpan{
Type: base.Select,
SourceColumns: accessesMap,
- PredicateColumns: e.funcPredicateColumns,
+ PredicateColumns: e.predicateColumns,
Results: results,
UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
}, nil
}
+func (e *omniQuerySpanExtractor) collectExplainedPredicateColumns(stmt ast.Node, accesses base.SourceColumnSet) error {
+ explain, ok := stmt.(*ast.ExplainStmt)
+ if !ok {
+ return nil
+ }
+ explained, ok := explain.Query.(*ast.SelectStmt)
+ if !ok {
+ return nil
+ }
+ if err := e.initCatalog(); err != nil {
+ return errors.Wrapf(err, "failed to initialize catalog")
+ }
+ if query, err := e.analyzeSelect(explained); err == nil {
+ e.collectStatementPredicateColumns(query, explained, accesses)
+ } else {
+ e.collectFallbackPredicateColumns(explained, accesses)
+ }
+ return nil
+}
+
+// mergeFunctionAccesses adds the tables the analyzed function bodies read to
+// accesses, so access checks and the unresolved-columns signal see them.
+func (e *omniQuerySpanExtractor) mergeFunctionAccesses(accesses base.SourceColumnSet) {
+ for col := range e.funcSourceColumns {
+ accesses[col] = true
+ }
+}
+
+// analyzeSelect analyzes sel for lineage. A function the synced schema does not
+// define, such as one from an extension, stays an opaque call over its
+// arguments instead of failing the statement into the parse-tree fallback.
+func (e *omniQuerySpanExtractor) analyzeSelect(sel *ast.SelectStmt) (*catalog.Query, error) {
+ return e.cat.AnalyzeSelectStmtWithOptions(sel, catalog.AnalyzeOptions{OpaqueUnknownFunctions: true})
+}
+
+// outputColumn returns the target entry of q's attNum-th output column. Junk
+// entries follow the output columns, so the position indexes the target list.
+func outputColumn(q *catalog.Query, attNum int) *catalog.TargetEntry {
+ if q == nil || attNum < 1 || attNum > len(q.TargetList) {
+ return nil
+ }
+ return q.TargetList[attNum-1]
+}
+
// extractLineage extracts the result column lineage from an analyzed query.
// selStmt is the original parse tree, used to determine IsPlainField
// (which is true only for columns that came from * expansion).
@@ -465,20 +528,19 @@ func isUltimatelyPlainColumnWithVisited(q *catalog.Query, expr catalog.AnalyzedE
return false
}
rte := q.RangeTable[v.RangeIdx]
- colIdx := int(v.AttNum - 1)
switch rte.Kind {
case catalog.RTERelation:
return true
case catalog.RTESubquery:
- if rte.Subquery != nil && colIdx >= 0 && colIdx < len(rte.Subquery.TargetList) {
- return isUltimatelyPlainColumnWithVisited(rte.Subquery, rte.Subquery.TargetList[colIdx].Expr, visited)
+ if te := outputColumn(rte.Subquery, int(v.AttNum)); te != nil {
+ return isUltimatelyPlainColumnWithVisited(rte.Subquery, te.Expr, visited)
}
case catalog.RTECTE:
if rte.CTEIndex >= 0 && rte.CTEIndex < len(q.CTEList) {
cte := q.CTEList[rte.CTEIndex]
- if cte.Query != nil && colIdx >= 0 && colIdx < len(cte.Query.TargetList) {
- return isUltimatelyPlainColumnWithVisited(cte.Query, cte.Query.TargetList[colIdx].Expr, visited)
+ if te := outputColumn(cte.Query, int(v.AttNum)); te != nil {
+ return isUltimatelyPlainColumnWithVisited(cte.Query, te.Expr, visited)
}
}
default:
@@ -696,7 +758,7 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Query
e.walkExprWithVisited(queryStack, arg, result, visited)
}
// Trace lineage through user-defined function body.
- if proc := e.cat.GetUserProcByOID(v.FuncOID); proc != nil {
+ for _, proc := range e.userProcsCalled(v) {
// Expression context: merge all output columns into one set.
for _, colSet := range e.analyzeFunctionBody(proc) {
for k, val := range colSet {
@@ -704,28 +766,6 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Query
}
}
}
- case *catalog.AggExpr:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
- }
- case *catalog.OpExpr:
- e.walkExprWithVisited(queryStack, v.Left, result, visited)
- e.walkExprWithVisited(queryStack, v.Right, result, visited)
- case *catalog.BoolExprQ:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
- }
- case *catalog.CaseExprQ:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- for _, w := range v.When {
- e.walkExprWithVisited(queryStack, w.Condition, result, visited)
- e.walkExprWithVisited(queryStack, w.Result, result, visited)
- }
- e.walkExprWithVisited(queryStack, v.Default, result, visited)
- case *catalog.CoalesceExprQ:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
- }
case *catalog.SubLinkExpr:
e.walkExprWithVisited(queryStack, v.TestExpr, result, visited)
if v.SubQuery != nil {
@@ -736,47 +776,21 @@ func (e *omniQuerySpanExtractor) walkExprWithVisited(queryStack []*catalog.Query
}
}
}
- case *catalog.RelabelExpr:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- case *catalog.CoerceViaIOExpr:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- case *catalog.RowExprQ:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
- }
- case *catalog.WindowFuncExpr:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
+ case *catalog.SetOpColumnExpr:
+ // q is the set operation. EXCEPT returns only rows of its left branch.
+ branches := []*catalog.Query{q.LArg, q.RArg}
+ if q.SetOp == catalog.SetOpExcept || q.SetOp == catalog.SetOpExceptAll {
+ branches = branches[:1]
}
- e.walkExprWithVisited(queryStack, v.AggFilter, result, visited)
- case *catalog.NullIfExprQ:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
- }
- case *catalog.MinMaxExprQ:
- for _, arg := range v.Args {
- e.walkExprWithVisited(queryStack, arg, result, visited)
+ for _, branch := range branches {
+ if te := outputColumn(branch, int(v.ResNo)); te != nil {
+ e.walkExprWithVisited(appendQuery(queryStack, branch), te.Expr, result, visited)
+ }
}
- case *catalog.ArrayExprQ:
- for _, elem := range v.Elements {
- e.walkExprWithVisited(queryStack, elem, result, visited)
- }
- case *catalog.ScalarArrayOpExpr:
- e.walkExprWithVisited(queryStack, v.Left, result, visited)
- e.walkExprWithVisited(queryStack, v.Right, result, visited)
- case *catalog.CollateExprQ:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- case *catalog.NullTestExpr:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- case *catalog.BooleanTestExpr:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- case *catalog.DistinctExprQ:
- e.walkExprWithVisited(queryStack, v.Left, result, visited)
- e.walkExprWithVisited(queryStack, v.Right, result, visited)
- case *catalog.FieldSelectExprQ:
- e.walkExprWithVisited(queryStack, v.Arg, result, visited)
- // ConstExpr, SQLValueFuncExpr, CoerceToDomainValueExpr — no column refs.
default:
+ for _, child := range catalog.AnalyzedExprChildren(expr) {
+ e.walkExprWithVisited(queryStack, child, result, visited)
+ }
}
}
@@ -840,32 +854,34 @@ func (e *omniQuerySpanExtractor) resolveVar(queryStack []*catalog.Query, v *cata
}] = true
case catalog.RTESubquery:
- if rte.Subquery == nil {
- return
- }
- if colIdx >= 0 && colIdx < len(rte.Subquery.TargetList) {
- te := rte.Subquery.TargetList[colIdx]
+ if te := outputColumn(rte.Subquery, int(v.AttNum)); te != nil {
e.walkExprWithVisited(appendQuery(effectiveStack, rte.Subquery), te.Expr, result, visited)
}
case catalog.RTECTE:
- if cte := findCTEForRangeTableEntry(effectiveStack, rte); cte != nil && cte.Query != nil && colIdx >= 0 && colIdx < len(cte.Query.TargetList) {
- te := cte.Query.TargetList[colIdx]
- e.walkExprWithVisited(appendQuery(effectiveStack, cte.Query), te.Expr, result, visited)
+ if cte := findCTEForRangeTableEntry(effectiveStack, rte); cte != nil {
+ if te := outputColumn(cte.Query, int(v.AttNum)); te != nil {
+ e.walkExprWithVisited(appendQuery(effectiveStack, cte.Query), te.Expr, result, visited)
+ }
}
case catalog.RTEJoin:
- // After analysis, VarExprs typically point to base RTEs, not join RTEs.
+ // Only a column that FULL JOIN USING merges is read through the join.
+ if colIdx >= 0 && colIdx < len(rte.JoinAliasVars) {
+ e.walkExprWithVisited(effectiveStack, rte.JoinAliasVars[colIdx], result, visited)
+ }
case catalog.RTEFunction:
// Trace lineage through user-defined function body.
if len(rte.FuncExprs) > 0 {
if fc, ok := rte.FuncExprs[0].(*catalog.FuncCallExpr); ok {
- if proc := e.cat.GetUserProcByOID(fc.FuncOID); proc != nil {
- bodySets := e.analyzeTableFunctionBody(proc)
- if colIdx >= 0 && colIdx < len(bodySets) {
- for k, val := range bodySets[colIdx] {
- result[k] = val
+ if procs := e.userProcsCalled(fc); len(procs) > 0 {
+ for _, proc := range procs {
+ bodySets := e.analyzeFunctionBody(proc)
+ if colIdx >= 0 && colIdx < len(bodySets) {
+ for k, val := range bodySets[colIdx] {
+ result[k] = val
+ }
}
}
} else {
@@ -962,7 +978,7 @@ func (e *omniQuerySpanExtractor) extractFallbackColumns(selStmt *ast.SelectStmt)
whereColSet := make(base.SourceColumnSet)
analyzer.extractColumnRefsFromExpr(selStmt.WhereClause, fromTables, whereColSet)
for k, v := range whereColSet {
- e.funcPredicateColumns[k] = v
+ e.predicateColumns[k] = v
}
}
return results
@@ -1368,7 +1384,7 @@ func (e *omniQuerySpanExtractor) extractColumnsFromRangeFunction(rf *ast.RangeFu
if proc := e.lookupUserProcByName(funcName); proc != nil {
outNames := getOutputParamNames(proc)
if len(outNames) > 0 {
- bodySets := e.analyzeTableFunctionBody(proc)
+ bodySets := e.analyzeFunctionBody(proc)
var results []base.QuerySpanResult
for i, name := range outNames {
colSet := make(base.SourceColumnSet)
@@ -1530,7 +1546,7 @@ func (e *omniQuerySpanExtractor) tryUserFuncTableSource(selStmt *ast.SelectStmt,
if len(outNames) == 0 {
continue
}
- bodySets := e.analyzeTableFunctionBody(proc)
+ bodySets := e.analyzeFunctionBody(proc)
return e.buildFuncResults(outNames, bodySets, accessesMap)
}
@@ -1677,7 +1693,7 @@ func (e *omniQuerySpanExtractor) tryMetadataFuncLookup(funcName string, accesses
// Use a temporary OID for caching.
syntheticProc.OID = uint32(0xFFFF0000 + len(e.funcBodyCache))
- bodySets := e.analyzeTableFunctionBody(syntheticProc)
+ bodySets := e.analyzeFunctionBody(syntheticProc)
return e.buildFuncResults(outNames, bodySets, accessesMap)
}
diff --git a/backend/plugin/parser/pg/query_span_plpgsql.go b/backend/plugin/parser/pg/query_span_plpgsql.go
index 34ed565fcf..315aa92115 100644
--- a/backend/plugin/parser/pg/query_span_plpgsql.go
+++ b/backend/plugin/parser/pg/query_span_plpgsql.go
@@ -13,6 +13,13 @@ import (
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
+// snapshotFunc is a function's language and body as the schema snapshot
+// defines it.
+type snapshotFunc struct {
+ language string
+ body string
+}
+
type funcBodyAnalysis struct {
sourceColumns []base.SourceColumnSet
predicateColumns base.SourceColumnSet
@@ -25,13 +32,7 @@ type funcBodyAnalysis struct {
func (e *omniQuerySpanExtractor) analyzeFunctionBody(proc *catalog.UserProc) []base.SourceColumnSet {
analysis := e.analyzeFunctionBodyForProc(proc)
e.mergeFunctionSourceTables(analysis.sourceColumns)
- mergeSourceColumnSetInto(e.funcPredicateColumns, analysis.predicateColumns)
- return analysis.sourceColumns
-}
-
-func (e *omniQuerySpanExtractor) analyzeTableFunctionBody(proc *catalog.UserProc) []base.SourceColumnSet {
- analysis := e.analyzeFunctionBodyForProc(proc)
- e.mergeFunctionSourceTables(analysis.sourceColumns)
+ mergeSourceColumnSetInto(e.predicateColumns, analysis.predicateColumns)
return analysis.sourceColumns
}
@@ -50,38 +51,51 @@ func (e *omniQuerySpanExtractor) analyzeFunctionBodyForProc(proc *catalog.UserPr
// Store nil sentinel to detect recursion.
e.funcBodyCache[proc.OID] = nil
- outerPredicateColumns := e.funcPredicateColumns
+ outerPredicateColumns := e.predicateColumns
bodyPredicateColumns := make(base.SourceColumnSet)
- e.funcPredicateColumns = bodyPredicateColumns
+ e.predicateColumns = bodyPredicateColumns
defer func() {
- e.funcPredicateColumns = outerPredicateColumns
+ e.predicateColumns = outerPredicateColumns
}()
- // If the function body was stubbed during catalog loading (to avoid type
- // validation errors), use the original body from metadata instead.
- body := proc.Body
- if origBody, ok := e.funcOrigDefs[strings.ToLower(proc.Name)]; ok && origBody != "" {
- body = origBody
+ // The catalog keeps each overload's own body. A function it could not
+ // install is a stand-in with a placeholder body, and the snapshot cannot
+ // tell which overload it stands for, so every overload's body is analyzed
+ // and merged.
+ defs := []snapshotFunc{{language: proc.Language, body: proc.Body}}
+ if proc.Schema != nil {
+ key := strings.ToLower(proc.Schema.Name + "." + proc.Name)
+ if e.standInFuncs[key] && len(e.snapshotFuncs[key]) > 0 {
+ defs = e.snapshotFuncs[key]
+ }
}
var result []base.SourceColumnSet
var err error
-
- switch strings.ToLower(proc.Language) {
- case "plpgsql":
- result, err = e.analyzePLpgSQLBody(proc, body)
- case "sql":
- result, err = e.analyzeSQLBody(body)
- default:
- // Unsupported language (C, internal, etc.) — unknown lineage.
- result = makeEmptySets(proc)
+ for _, def := range defs {
+ defResult, defErr := e.analyzeBodyAs(proc, def)
+ if defErr != nil {
+ err = defErr
+ continue
+ }
+ if len(defs) == 1 {
+ result = defResult
+ break
+ }
+ for i, set := range defResult {
+ if i >= len(result) {
+ result = append(result, make(base.SourceColumnSet))
+ }
+ mergeSourceColumnSetInto(result[i], set)
+ }
}
- if err != nil {
- // On error, cache empty sets so we don't retry.
+ if err != nil && result == nil {
+ // On error, cache empty sets so we don't retry. The conditions found
+ // before the error still run, so they stay.
e.funcBodyCache[proc.OID] = &funcBodyAnalysis{
sourceColumns: makeEmptySets(proc),
- predicateColumns: make(base.SourceColumnSet),
+ predicateColumns: cloneSourceColumnSet(bodyPredicateColumns),
}
return e.funcBodyCache[proc.OID]
}
@@ -93,6 +107,18 @@ func (e *omniQuerySpanExtractor) analyzeFunctionBodyForProc(proc *catalog.UserPr
return e.funcBodyCache[proc.OID]
}
+func (e *omniQuerySpanExtractor) analyzeBodyAs(proc *catalog.UserProc, def snapshotFunc) ([]base.SourceColumnSet, error) {
+ switch strings.ToLower(def.language) {
+ case "plpgsql":
+ return e.analyzePLpgSQLBody(proc, def.body)
+ case "sql":
+ return e.analyzeSQLBody(def.body)
+ default:
+ // Unsupported language (C, internal, etc.) — unknown lineage.
+ return makeEmptySets(proc), nil
+ }
+}
+
func cloneSourceColumnSet(src base.SourceColumnSet) base.SourceColumnSet {
dst := make(base.SourceColumnSet)
for k, v := range src {
@@ -152,30 +178,64 @@ func (e *omniQuerySpanExtractor) analyzeSQLBody(body string) ([]base.SourceColum
return nil, errors.Wrapf(err, "failed to parse SQL function body")
}
- // Find the last SELECT statement (SQL functions return the result of the last statement).
- for i := len(stmts) - 1; i >= 0; i-- {
- selStmt, ok := stmts[i].AST.(*ast.SelectStmt)
+ // Every statement in the body runs, so each one's conditions count; the
+ // last one returns the function's result.
+ last := -1
+ for i, stmt := range stmts {
+ if stmt.AST != nil {
+ last = i
+ }
+ }
+ var result []base.SourceColumnSet
+ for i, stmt := range stmts {
+ if stmt.AST == nil {
+ continue
+ }
+ accesses := e.accessTablesOf(stmt.Text)
+ // The caller reads what the body reads, whether or not the result shows it.
+ mergeSourceColumnSetInto(e.funcSourceColumns, accesses)
+ selStmt, ok := stmt.AST.(*ast.SelectStmt)
if !ok {
+ // The analyzer reads SELECT alone, so another statement's conditions
+ // and RETURNING list are matched by name.
+ e.collectFallbackPredicateColumns(stmt.AST, accesses)
+ returning := e.returningLineage(stmt.AST, accesses)
+ if i == last {
+ result = returning
+ } else {
+ for _, set := range returning {
+ mergeSourceColumnSetInto(e.predicateColumns, set)
+ }
+ }
continue
}
- query, err := e.cat.AnalyzeSelectStmt(selStmt)
+ query, err := e.analyzeSelect(selStmt)
if err != nil {
+ e.collectFallbackPredicateColumns(selStmt, accesses)
+ if i != last {
+ e.collectDiscardedOutputColumns(nil, selStmt, accesses)
+ continue
+ }
// Fall back to parse-tree extraction when analysis fails
// (e.g., nested function calls, parameter references).
fallbackResults := e.extractFallbackColumns(selStmt)
- if len(fallbackResults) > 0 {
- result := make([]base.SourceColumnSet, len(fallbackResults))
- for j, fr := range fallbackResults {
- result[j] = fr.SourceColumns
- }
- return result, nil
+ if len(fallbackResults) == 0 {
+ return nil, errors.Wrapf(err, "failed to analyze SQL function body")
}
- return nil, errors.Wrapf(err, "failed to analyze SQL function body")
+ result = make([]base.SourceColumnSet, len(fallbackResults))
+ for j, fr := range fallbackResults {
+ result[j] = fr.SourceColumns
+ }
+ continue
+ }
+ e.collectStatementPredicateColumns(query, selStmt, accesses)
+ if i == last {
+ result = e.extractFuncLineage(query)
+ } else {
+ e.collectDiscardedOutputColumns(query, selStmt, accesses)
}
- return e.extractFuncLineage(query), nil
}
-
- return nil, nil
+ return result, nil
}
func (e *omniQuerySpanExtractor) extractFuncLineage(q *catalog.Query) []base.SourceColumnSet {
@@ -306,9 +366,14 @@ func (a *plpgsqlAnalyzer) walkBlock(block *plpgsqlast.PLBlock) {
for _, decl := range block.Declarations {
switch d := decl.(type) {
case *plpgsqlast.PLDeclare:
- a.scope.set(d.Name, make(base.SourceColumnSet))
+ sources := make(base.SourceColumnSet)
+ if d.Default != "" {
+ sources = a.analyzeEmbeddedExpr(d.Default)
+ }
+ a.scope.set(d.Name, sources)
case *plpgsqlast.PLCursorDecl:
- a.scope.set(d.Name, make(base.SourceColumnSet))
+ // A cursor carries the columns of its query to what FETCH reads.
+ a.scope.set(d.Name, a.analyzeQuerySources(d.Query))
case *plpgsqlast.PLAliasDecl:
// Alias references another variable — copy its sources.
if sources, ok := a.scope.get(d.RefName); ok {
@@ -356,16 +421,23 @@ func (a *plpgsqlAnalyzer) walkStmt(node plpgsqlast.Node) {
a.walkStmt(s)
}
case *plpgsqlast.PLWhile:
+ a.handleCondition(stmt.Condition)
for _, s := range stmt.Body {
a.walkStmt(s)
}
case *plpgsqlast.PLForI:
+ a.handleCondition(stmt.Lower)
+ a.handleCondition(stmt.Upper)
+ a.handleCondition(stmt.Step)
for _, s := range stmt.Body {
a.walkStmt(s)
}
case *plpgsqlast.PLForS:
a.handleForS(stmt)
case *plpgsqlast.PLForC:
+ if sources, ok := a.scope.get(stmt.CursorVar); ok {
+ a.scope.merge(stmt.Var, sources)
+ }
for _, s := range stmt.Body {
a.walkStmt(s)
}
@@ -374,18 +446,124 @@ func (a *plpgsqlAnalyzer) walkStmt(node plpgsqlast.Node) {
a.walkStmt(s)
}
case *plpgsqlast.PLForEachA:
+ a.handleCondition(stmt.ArrayExpr)
for _, s := range stmt.Body {
a.walkStmt(s)
}
case *plpgsqlast.PLBlock:
a.walkBlock(stmt)
+ // PERFORM discards its rows but still runs, so how long it takes reveals
+ // what it reads.
+ case *plpgsqlast.PLPerform:
+ a.handleCondition(stmt.Expr)
+ case *plpgsqlast.PLDynExecute:
+ a.handleDynExecute(stmt)
+ case *plpgsqlast.PLOpen:
+ if stmt.Query != "" {
+ a.scope.merge(stmt.CursorVar, a.analyzeQuerySources(stmt.Query))
+ }
+ case *plpgsqlast.PLFetch:
+ if sources, ok := a.scope.get(stmt.CursorVar); ok {
+ for _, varName := range stmt.Into {
+ a.scope.merge(varName, sources)
+ }
+ }
+ case *plpgsqlast.PLCall:
+ a.handleCall(stmt)
+ // RAISE puts the values of its parameters in the message the caller sees.
+ case *plpgsqlast.PLRaise:
+ for _, param := range stmt.Params {
+ a.handleCondition(param)
+ }
+ case *plpgsqlast.PLExit:
+ a.handleCondition(stmt.Condition)
+ case *plpgsqlast.PLAssert:
+ a.handleCondition(stmt.Condition)
// Skip nodes that don't affect lineage:
- // PLDynExecute, PLRaise, PLNull, PLCommit, PLRollback, PLPerform,
- // PLCall, PLOpen, PLFetch, PLClose, PLGetDiag, PLAssert, PLExit
+ // PLNull, PLCommit, PLRollback, PLClose, PLGetDiag
default:
}
}
+// handleCondition records the columns an expression that steers the body
+// reads, directly or through variables: it decides which statements run, and
+// so what the function returns and how long it takes.
+func (a *plpgsqlAnalyzer) handleCondition(expr string) {
+ if strings.TrimSpace(expr) == "" {
+ return
+ }
+ mergeSourceColumnSetInto(a.extractor.predicateColumns, a.analyzeEmbeddedExpr(expr))
+ stmts, err := ParsePg("SELECT " + expr)
+ if err != nil || len(stmts) != 1 {
+ return
+ }
+ a.addScopeReferences(stmts[0].AST)
+}
+
+// addScopeReferences records, as predicates, the columns that the variables
+// node reads carry.
+func (a *plpgsqlAnalyzer) addScopeReferences(node ast.Node) {
+ mergeSourceColumnSetInto(a.extractor.predicateColumns, a.scopeSources(node))
+}
+
+// scopeSources returns the columns that the variables node reads carry. A
+// variable, or the record variable a qualified reference such as r.ssn names,
+// carries the columns it was assigned from.
+func (a *plpgsqlAnalyzer) scopeSources(node ast.Node) base.SourceColumnSet {
+ sources := make(base.SourceColumnSet)
+ ast.Inspect(node, func(n ast.Node) bool {
+ if ref, ok := n.(*ast.ColumnRef); ok && ref.Fields != nil && len(ref.Fields.Items) > 0 {
+ if name, ok := ref.Fields.Items[0].(*ast.String); ok {
+ if varSources, ok := a.scope.get(name.Str); ok {
+ mergeSourceColumnSetInto(sources, varSources)
+ }
+ }
+ }
+ return true
+ })
+ return sources
+}
+
+// analyzeQuerySources returns every column the output of sql reads, merged.
+func (a *plpgsqlAnalyzer) analyzeQuerySources(sql string) base.SourceColumnSet {
+ merged := make(base.SourceColumnSet)
+ if strings.TrimSpace(sql) == "" {
+ return merged
+ }
+ for _, set := range a.analyzeEmbeddedSQL(sql) {
+ mergeSourceColumnSetInto(merged, set)
+ }
+ return merged
+}
+
+// handleCall analyzes the body of the procedure CALL invokes, whatever
+// overload it is. The procedure runs for its effects, so the values passed to
+// it count as predicates.
+func (a *plpgsqlAnalyzer) handleCall(stmt *plpgsqlast.PLCall) {
+ stmts, err := ParsePg(stmt.SQLText)
+ if err != nil || len(stmts) != 1 {
+ return
+ }
+ call, ok := stmts[0].AST.(*ast.CallStmt)
+ if !ok || call.Funccall == nil {
+ return
+ }
+ for _, proc := range a.extractor.userProcsNamed(astStrings(call.Funccall.Funcname)) {
+ a.extractor.analyzeFunctionBody(proc)
+ }
+ if call.Funccall.Args == nil {
+ return
+ }
+ for _, arg := range call.Funccall.Args.Items {
+ loc := ast.NodeLoc(arg)
+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(stmt.SQLText) {
+ a.handleCondition(stmt.SQLText[loc.Start:loc.End])
+ continue
+ }
+ a.addScopeReferences(arg)
+ }
+}
+
func (a *plpgsqlAnalyzer) handleAssign(stmt *plpgsqlast.PLAssign) {
varName := extractBaseVarName(stmt.Target)
sources := a.analyzeEmbeddedExpr(stmt.Expr)
@@ -419,10 +597,20 @@ func (a *plpgsqlAnalyzer) handleReturnNext(stmt *plpgsqlast.PLReturnNext) {
}
func (a *plpgsqlAnalyzer) handleExecSQL(stmt *plpgsqlast.PLExecSQL) {
- if len(stmt.Into) == 0 {
+ stmts, err := ParsePg(stmt.SQLText)
+ if err != nil || len(stmts) != 1 {
return
}
- results := a.analyzeEmbeddedSQL(stmt.SQLText)
+ var results []base.SourceColumnSet
+ if _, ok := stmts[0].AST.(*ast.SelectStmt); ok {
+ results = a.analyzeEmbeddedSQL(stmt.SQLText)
+ } else {
+ // The analyzer reads SELECT alone, so another statement, which runs for
+ // its effect, has its conditions and RETURNING list matched by name.
+ accesses := a.extractor.accessTablesOf(stmt.SQLText)
+ a.extractor.collectFallbackPredicateColumns(stmts[0].AST, accesses)
+ results = a.extractor.returningLineage(stmts[0].AST, accesses)
+ }
for i, varName := range stmt.Into {
if i < len(results) {
a.scope.merge(varName, results[i])
@@ -430,6 +618,55 @@ func (a *plpgsqlAnalyzer) handleExecSQL(stmt *plpgsqlast.PLExecSQL) {
}
}
+// handleDynExecute analyzes EXECUTE of a constant query like the statement it
+// runs. A query built at run time cannot be analyzed.
+func (a *plpgsqlAnalyzer) handleDynExecute(stmt *plpgsqlast.PLDynExecute) {
+ stmts, err := ParsePg("SELECT " + stmt.Query)
+ if err != nil || len(stmts) != 1 {
+ return
+ }
+ sel, ok := stmts[0].AST.(*ast.SelectStmt)
+ if !ok || sel.TargetList == nil || len(sel.TargetList.Items) != 1 {
+ return
+ }
+ rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)
+ if !ok {
+ return
+ }
+ query, ok := constantString(rt.Val)
+ if !ok {
+ return
+ }
+ a.handleExecSQL(&plpgsqlast.PLExecSQL{SQLText: query, Into: stmt.Into})
+}
+
+// constantString returns the text of a string literal, a cast of one, or a
+// concatenation of them.
+func constantString(node ast.Node) (string, bool) {
+ switch v := node.(type) {
+ case *ast.A_Const:
+ if s, ok := v.Val.(*ast.String); ok {
+ return s.Str, true
+ }
+ case *ast.TypeCast:
+ return constantString(v.Arg)
+ case *ast.A_Expr:
+ if names := astStrings(v.Name); len(names) == 1 && names[0] == "||" {
+ left, ok := constantString(v.Lexpr)
+ if !ok {
+ return "", false
+ }
+ right, ok := constantString(v.Rexpr)
+ if !ok {
+ return "", false
+ }
+ return left + right, true
+ }
+ default:
+ }
+ return "", false
+}
+
func (a *plpgsqlAnalyzer) handleForS(stmt *plpgsqlast.PLForS) {
results := a.analyzeEmbeddedSQL(stmt.Query)
merged := make(base.SourceColumnSet)
@@ -445,11 +682,13 @@ func (a *plpgsqlAnalyzer) handleForS(stmt *plpgsqlast.PLForS) {
}
func (a *plpgsqlAnalyzer) handleIf(stmt *plpgsqlast.PLIf) {
+ a.handleCondition(stmt.Condition)
for _, s := range stmt.ThenBody {
a.walkStmt(s)
}
for _, elsif := range stmt.ElsIfs {
if ei, ok := elsif.(*plpgsqlast.PLElsIf); ok {
+ a.handleCondition(ei.Condition)
for _, s := range ei.Body {
a.walkStmt(s)
}
@@ -461,8 +700,10 @@ func (a *plpgsqlAnalyzer) handleIf(stmt *plpgsqlast.PLIf) {
}
func (a *plpgsqlAnalyzer) handleCase(stmt *plpgsqlast.PLCase) {
+ a.handleCondition(stmt.TestExpr)
for _, when := range stmt.Whens {
if cw, ok := when.(*plpgsqlast.PLCaseWhen); ok {
+ a.handleCondition(cw.Expr)
for _, s := range cw.Body {
a.walkStmt(s)
}
@@ -478,7 +719,7 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSet
// Always collect access tables from embedded SQL for the top-level QuerySpan,
// regardless of whether full analysis succeeds.
- a.collectAccessTablesFromSQL(substituted)
+ accesses := a.collectAccessTablesFromSQL(substituted)
stmts, err := ParsePg(substituted)
if err != nil {
@@ -493,17 +734,16 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSet
return nil
}
- query, err := a.extractor.cat.AnalyzeSelectStmt(selStmt)
+ query, err := a.extractor.analyzeSelect(selStmt)
if err != nil {
// When full analysis fails (e.g., type mismatches with aggregate functions),
// fall back to extracting column references from the parse tree and resolving
// them against known tables in the catalog.
+ a.extractor.collectFallbackPredicateColumns(selStmt, accesses)
return a.fallbackExtractLineage(sql, selStmt)
}
- // Also collect from the analyzed query for precise column-level tracking
- // (WHERE clause → predicate columns).
- a.collectQueryPredicateColumns(query)
+ a.extractor.collectStatementPredicateColumns(query, selStmt, accesses)
results := a.extractor.extractFuncLineage(query)
@@ -586,11 +826,8 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedExpr(expr string) base.SourceColumnSet
}
}
if len(merged) == 0 {
- varName := strings.TrimSpace(expr)
- if sources, ok := a.scope.get(varName); ok {
- for k, v := range sources {
- merged[k] = v
- }
+ if stmts, err := ParsePg("SELECT " + expr); err == nil && len(stmts) == 1 {
+ mergeSourceColumnSetInto(merged, a.scopeSources(stmts[0].AST))
}
}
return merged
@@ -600,7 +837,7 @@ func (a *plpgsqlAnalyzer) substituteVariables(sql string) string {
stmts, err := ParsePg(sql)
if err == nil && len(stmts) > 0 {
if selStmt, ok := stmts[0].AST.(*ast.SelectStmt); ok {
- if _, err := a.extractor.cat.AnalyzeSelectStmt(selStmt); err == nil {
+ if _, err := a.extractor.analyzeSelect(selStmt); err == nil {
return sql
}
}
@@ -659,74 +896,36 @@ func extractBaseVarName(target string) string {
return target
}
-// collectAccessTablesFromSQL extracts all table/column references from embedded SQL
-// and adds them to the extractor's funcSourceColumns. This works even when
-// AnalyzeSelectStmt fails, because it uses the AST-level ExtractAccessTables.
-func (a *plpgsqlAnalyzer) collectAccessTablesFromSQL(sql string) {
+// collectAccessTablesFromSQL extracts all table/column references from embedded SQL,
+// adds them to the extractor's funcSourceColumns, and returns them. This works even
+// when AnalyzeSelectStmt fails, because it uses the AST-level ExtractAccessTables.
+func (a *plpgsqlAnalyzer) collectAccessTablesFromSQL(sql string) base.SourceColumnSet {
+ accesses := a.extractor.accessTablesOf(sql)
+ for resource := range accesses {
+ a.extractor.funcSourceColumns[resource] = true
+ }
+ return accesses
+}
+
+// accessTablesOf returns the tables sql reads, or none when it cannot tell.
+func (e *omniQuerySpanExtractor) accessTablesOf(sql string) base.SourceColumnSet {
+ accesses := make(base.SourceColumnSet)
accessTables, err := ExtractAccessTables(sql, ExtractAccessTablesOption{
- DefaultDatabase: a.extractor.defaultDatabase,
- DefaultSchema: a.extractor.searchPath[0],
- SearchPath: a.extractor.searchPath,
- GetDatabaseMetadata: a.extractor.gCtx.GetDatabaseMetadataFunc,
- Ctx: a.extractor.ctx,
- InstanceID: a.extractor.gCtx.InstanceID,
+ DefaultDatabase: e.defaultDatabase,
+ DefaultSchema: e.searchPath[0],
+ SearchPath: e.searchPath,
+ GetDatabaseMetadata: e.gCtx.GetDatabaseMetadataFunc,
+ Ctx: e.ctx,
+ InstanceID: e.gCtx.InstanceID,
SkipMetadataValidation: false,
})
if err != nil {
- return
+ return accesses
}
for _, resource := range accessTables {
- a.extractor.funcSourceColumns[resource] = true
- }
-}
-
-// collectQueryPredicateColumns walks WHERE/HAVING clauses in an analyzed query
-// and collects predicate columns (column-level) into funcPredicateColumns.
-// Note: funcSourceColumns only stores table-level access (via collectAccessTablesFromSQL),
-// so we don't add column-level entries to it here.
-func (a *plpgsqlAnalyzer) collectQueryPredicateColumns(q *catalog.Query) {
- a.collectQueryPredicateColumnsWithVisited(q, make(map[*catalog.Query]bool))
-}
-
-func (a *plpgsqlAnalyzer) collectQueryPredicateColumnsWithVisited(q *catalog.Query, visited map[*catalog.Query]bool) {
- if q == nil {
- return
- }
- if visited[q] {
- return
- }
- visited[q] = true
-
- // Handle set operations recursively.
- if q.SetOp != catalog.SetOpNone {
- a.collectQueryPredicateColumnsWithVisited(q.LArg, visited)
- a.collectQueryPredicateColumnsWithVisited(q.RArg, visited)
- return
- }
-
- // Collect from WHERE clause → predicate columns only.
- if q.JoinTree != nil && q.JoinTree.Quals != nil {
- a.extractor.walkExpr(q, q.JoinTree.Quals, a.extractor.funcPredicateColumns)
- }
-
- // Collect from HAVING clause → predicate columns only.
- if q.HavingQual != nil {
- a.extractor.walkExpr(q, q.HavingQual, a.extractor.funcPredicateColumns)
- }
-
- // Recurse into CTEs.
- for _, cte := range q.CTEList {
- if cte.Query != nil {
- a.collectQueryPredicateColumnsWithVisited(cte.Query, visited)
- }
- }
-
- // Recurse into subqueries in FROM clause.
- for _, rte := range q.RangeTable {
- if rte.Kind == catalog.RTESubquery && rte.Subquery != nil {
- a.collectQueryPredicateColumnsWithVisited(rte.Subquery, visited)
- }
+ accesses[resource] = true
}
+ return accesses
}
func (a *plpgsqlAnalyzer) mergeReturnResults(nCols int) []base.SourceColumnSet {
@@ -796,7 +995,7 @@ func (a *plpgsqlAnalyzer) fallbackExtractLineage(origSQL string, selStmt *ast.Se
whereColSet := make(base.SourceColumnSet)
a.extractColumnRefsFromExpr(selStmt.WhereClause, fromTables, whereColSet)
for k, v := range whereColSet {
- a.extractor.funcPredicateColumns[k] = v
+ a.extractor.predicateColumns[k] = v
}
}
@@ -958,7 +1157,7 @@ func (a *plpgsqlAnalyzer) extractColumnRefsFromExpr(node ast.Node, fromTables ma
filterColSet := make(base.SourceColumnSet)
a.extractColumnRefsFromExpr(v.AggFilter, fromTables, filterColSet)
for k, val := range filterColSet {
- a.extractor.funcPredicateColumns[k] = val
+ a.extractor.predicateColumns[k] = val
}
}
case *ast.A_Expr:
@@ -989,7 +1188,7 @@ func (a *plpgsqlAnalyzer) extractColumnRefsFromExpr(node ast.Node, fromTables ma
whereColSet := make(base.SourceColumnSet)
a.extractColumnRefsFromExpr(subSel.WhereClause, subTables, whereColSet)
for k, val := range whereColSet {
- a.extractor.funcPredicateColumns[k] = val
+ a.extractor.predicateColumns[k] = val
}
}
}
@@ -1211,7 +1410,7 @@ func (a *plpgsqlAnalyzer) resolveThroughCTE(cteSel *ast.SelectStmt, colName stri
whereColSet := make(base.SourceColumnSet)
a.extractColumnRefsFromExpr(cteSel.WhereClause, cteTables, whereColSet)
for k, v := range whereColSet {
- a.extractor.funcPredicateColumns[k] = v
+ a.extractor.predicateColumns[k] = v
}
}
return
diff --git a/backend/plugin/parser/pg/query_span_predicate.go b/backend/plugin/parser/pg/query_span_predicate.go
new file mode 100644
index 0000000000..cbb176fb9c
--- /dev/null
+++ b/backend/plugin/parser/pg/query_span_predicate.go
@@ -0,0 +1,651 @@
+package pg
+
+import (
+ "strings"
+
+ "github.com/bytebase/omni/pg/ast"
+ "github.com/bytebase/omni/pg/catalog"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// collectPredicateColumns records the columns that decide which rows q and
+// every query nested in it return: those read by WHERE, JOIN ON, HAVING, and
+// aggregate FILTER conditions, LIMIT and OFFSET, and the arguments of
+// set-returning functions. The database evaluates conditions on unmasked values, so these columns
+// decide which rows come back even when their values are masked.
+func (e *omniQuerySpanExtractor) collectPredicateColumns(q *catalog.Query) {
+ e.collectPredicateColumnsInStack([]*catalog.Query{q}, make(map[*catalog.Query]bool))
+}
+
+// collectStatementPredicateColumns records the predicate columns of sel,
+// analyzed as query, that reads accesses.
+func (e *omniQuerySpanExtractor) collectStatementPredicateColumns(query *catalog.Query, sel *ast.SelectStmt, accesses base.SourceColumnSet) {
+ e.collectPredicateColumns(query)
+ // The analyzer leaves a set operation's LIMIT and OFFSET out of its tree,
+ // so they are resolved by name.
+ r := newPredicateNameResolver(sel, e.namedFunctionReturnsSet)
+ limited := false
+ ast.Inspect(sel, func(n ast.Node) bool {
+ if v, ok := n.(*ast.SelectStmt); ok && v.Op != ast.SETOP_NONE {
+ for _, limit := range []ast.Node{v.LimitCount, v.LimitOffset} {
+ if limit != nil {
+ r.addReferences(limit)
+ e.analyzeFunctionsCalledIn(limit)
+ limited = true
+ }
+ }
+ }
+ return true
+ })
+ if limited {
+ e.recordPredicateNames(r, accesses)
+ }
+}
+
+// collectDiscardedOutputColumns records the columns a SELECT whose rows are
+// discarded reads in its output. It still evaluates the output, so those
+// columns decide how long it takes. query is nil when sel failed analysis.
+func (e *omniQuerySpanExtractor) collectDiscardedOutputColumns(query *catalog.Query, sel *ast.SelectStmt, accesses base.SourceColumnSet) {
+ if query != nil {
+ for _, set := range e.extractFuncLineage(query) {
+ mergeSourceColumnSetInto(e.predicateColumns, set)
+ }
+ return
+ }
+ if sel.TargetList == nil {
+ return
+ }
+ r := newPredicateNameResolver(sel, e.namedFunctionReturnsSet)
+ r.addReferences(sel.TargetList)
+ e.recordPredicateNames(r, accesses)
+}
+
+func (e *omniQuerySpanExtractor) collectPredicateColumnsInStack(queryStack []*catalog.Query, visited map[*catalog.Query]bool) {
+ q := currentQuery(queryStack)
+ if q == nil || visited[q] {
+ return
+ }
+ visited[q] = true
+
+ for _, cte := range q.CTEList {
+ if cte != nil {
+ e.collectPredicateColumnsInStack(appendQuery(queryStack, cte.Query), visited)
+ }
+ }
+ e.collectConditionColumns(queryStack, q.LimitCount, visited)
+ e.collectConditionColumns(queryStack, q.LimitOffset, visited)
+ if q.SetOp != catalog.SetOpNone {
+ // INTERSECT and EXCEPT keep a row by whether the other branch has it,
+ // so their branches' output columns decide which rows come back.
+ filters := q.SetOp == catalog.SetOpIntersect || q.SetOp == catalog.SetOpIntersectAll ||
+ q.SetOp == catalog.SetOpExcept || q.SetOp == catalog.SetOpExceptAll
+ for _, branch := range []*catalog.Query{q.LArg, q.RArg} {
+ if filters && branch != nil {
+ for _, te := range branch.TargetList {
+ if te != nil && !te.ResJunk {
+ e.walkExprWithVisited(appendQuery(queryStack, branch), te.Expr, e.predicateColumns, make(map[queryExprKey]bool))
+ }
+ }
+ }
+ e.collectPredicateColumnsInStack(appendQuery(queryStack, branch), visited)
+ }
+ return
+ }
+ for _, rte := range q.RangeTable {
+ if rte == nil {
+ continue
+ }
+ if rte.Kind == catalog.RTESubquery {
+ e.collectPredicateColumnsInStack(appendQuery(queryStack, rte.Subquery), visited)
+ }
+ for _, expr := range rte.FuncExprs {
+ // A function in FROM returns the rows its arguments and the
+ // conditions in its body ask for.
+ call, ok := expr.(*catalog.FuncCallExpr)
+ if !ok {
+ e.collectConditionColumns(queryStack, expr, visited)
+ continue
+ }
+ for _, arg := range call.Args {
+ e.collectConditionColumns(queryStack, arg, visited)
+ }
+ for _, proc := range e.userProcsCalled(call) {
+ e.analyzeFunctionBody(proc)
+ }
+ }
+ }
+ if q.JoinTree != nil {
+ e.collectConditionColumns(queryStack, q.JoinTree.Quals, visited)
+ for _, node := range q.JoinTree.FromList {
+ e.collectJoinConditionColumns(queryStack, node, visited)
+ }
+ }
+ e.collectConditionColumns(queryStack, q.HavingQual, visited)
+ for _, te := range q.TargetList {
+ if te != nil {
+ e.collectNestedPredicateColumns(queryStack, te.Expr, visited)
+ }
+ }
+}
+
+func (e *omniQuerySpanExtractor) collectJoinConditionColumns(queryStack []*catalog.Query, node catalog.JoinNode, visited map[*catalog.Query]bool) {
+ join, ok := node.(*catalog.JoinExprNode)
+ if !ok || join == nil {
+ return
+ }
+ // The analyzer turns USING and NATURAL joins into equality quals.
+ e.collectConditionColumns(queryStack, join.Quals, visited)
+ e.collectJoinConditionColumns(queryStack, join.Left, visited)
+ e.collectJoinConditionColumns(queryStack, join.Right, visited)
+}
+
+// collectConditionColumns records every column condition reads, including the
+// output columns of its subqueries, and the conditions nested in it.
+func (e *omniQuerySpanExtractor) collectConditionColumns(queryStack []*catalog.Query, condition catalog.AnalyzedExpr, visited map[*catalog.Query]bool) {
+ if condition == nil {
+ return
+ }
+ e.walkExprWithVisited(queryStack, condition, e.predicateColumns, make(map[queryExprKey]bool))
+ e.collectNestedPredicateColumns(queryStack, condition, visited)
+}
+
+// collectNestedPredicateColumns records the conditions nested in expr: the
+// FILTER of an aggregate or window function, the arguments of a set-returning
+// function, which decide how many rows it returns, and the conditions of its
+// subqueries.
+func (e *omniQuerySpanExtractor) collectNestedPredicateColumns(queryStack []*catalog.Query, expr catalog.AnalyzedExpr, visited map[*catalog.Query]bool) {
+ catalog.WalkAnalyzedExpr(expr, func(n catalog.AnalyzedExpr) bool {
+ var inputs []catalog.AnalyzedExpr
+ switch v := n.(type) {
+ case *catalog.SubLinkExpr:
+ if v.SubQuery != nil {
+ e.collectPredicateColumnsInStack(appendQuery(queryStack, v.SubQuery), visited)
+ }
+ case *catalog.AggExpr:
+ inputs = []catalog.AnalyzedExpr{v.AggFilter}
+ case *catalog.WindowFuncExpr:
+ inputs = []catalog.AnalyzedExpr{v.AggFilter}
+ case *catalog.FuncCallExpr:
+ if e.returnsSet(v.FuncOID) {
+ inputs = v.Args
+ }
+ default:
+ }
+ for _, input := range inputs {
+ if input != nil {
+ e.walkExprWithVisited(queryStack, input, e.predicateColumns, make(map[queryExprKey]bool))
+ }
+ }
+ return true
+ })
+}
+
+// analyzeFunctionsCalledIn analyzes the body of every user function node calls,
+// which merges the body's predicates, as walkExpr does for an analyzed call.
+// Without argument types the overload is unknown, so every one of the name is
+// analyzed.
+func (e *omniQuerySpanExtractor) analyzeFunctionsCalledIn(node ast.Node) {
+ ast.Inspect(node, func(n ast.Node) bool {
+ if call, ok := n.(*ast.FuncCall); ok {
+ for _, proc := range e.userProcsNamed(astStrings(call.Funcname)) {
+ e.analyzeFunctionBody(proc)
+ }
+ }
+ return true
+ })
+}
+
+// userProcsCalled returns the user function an analyzed call invokes. An
+// opaque call, which the analyzer could not match to a function (as for a
+// stand-in), may invoke any user function of its name.
+func (e *omniQuerySpanExtractor) userProcsCalled(call *catalog.FuncCallExpr) []*catalog.UserProc {
+ if proc := e.cat.GetUserProcByOID(call.FuncOID); proc != nil {
+ return []*catalog.UserProc{proc}
+ }
+ if call.FuncOID == 0 {
+ return e.userProcsNamed([]string{call.FuncName})
+ }
+ return nil
+}
+
+// userProcsNamed returns every user function, overloads included, that a
+// possibly schema-qualified name may call.
+func (e *omniQuerySpanExtractor) userProcsNamed(names []string) []*catalog.UserProc {
+ if len(names) == 0 {
+ return nil
+ }
+ name := names[len(names)-1]
+ schemas := e.searchPath
+ if len(names) > 1 {
+ schemas = []string{names[len(names)-2]}
+ }
+ var procs []*catalog.UserProc
+ for _, schema := range schemas {
+ for _, row := range e.cat.QueryPgProc(schema) {
+ if strings.EqualFold(row.ProName, name) {
+ if proc := e.cat.GetUserProcByOID(row.OID); proc != nil {
+ procs = append(procs, proc)
+ }
+ }
+ }
+ }
+ return procs
+}
+
+// namedFunctionReturnsSet reports whether a function of that name is known to
+// return a set.
+func (e *omniQuerySpanExtractor) namedFunctionReturnsSet(name string) bool {
+ for _, proc := range e.userProcsNamed([]string{name}) {
+ if proc.RetSet {
+ return true
+ }
+ }
+ for _, proc := range e.cat.LookupProcByName(strings.ToLower(name)) {
+ if proc.RetSet {
+ return true
+ }
+ }
+ return false
+}
+
+// returnsSet reports whether the function is known to return a set. An
+// unknown function, analyzed opaquely, is not.
+func (e *omniQuerySpanExtractor) returnsSet(oid uint32) bool {
+ if proc := e.cat.GetUserProcByOID(oid); proc != nil {
+ return proc.RetSet
+ }
+ if proc := e.cat.LookupProcByOID(oid); proc != nil {
+ return proc.RetSet
+ }
+ return false
+}
+
+// collectFallbackPredicateColumns stands in for collectPredicateColumns when
+// analysis fails. Without scopes it over-approximates by name, so that an
+// analysis failure never lets a condition on a masked column through.
+func (e *omniQuerySpanExtractor) collectFallbackPredicateColumns(stmt ast.Node, accesses base.SourceColumnSet) {
+ e.analyzeFunctionsCalledIn(stmt)
+ r := newPredicateNameResolver(stmt, e.namedFunctionReturnsSet)
+ for _, name := range r.usingNames {
+ r.addName(name)
+ }
+ for _, condition := range r.conditions {
+ r.addReferences(condition)
+ }
+ // The joined columns are unknown without scopes, so a NATURAL JOIN reads
+ // every column of both sides.
+ for _, join := range r.naturalJoins {
+ r.addJoinedRelation(join.Larg)
+ r.addJoinedRelation(join.Rarg)
+ }
+ for _, setOperation := range r.filteringSetOperations {
+ r.addSetOperationOutputs(setOperation)
+ }
+ e.recordPredicateNames(r, accesses)
+}
+
+func (e *omniQuerySpanExtractor) recordPredicateNames(r *predicateNameResolver, accesses base.SourceColumnSet) {
+ mergeSourceColumnSetInto(e.predicateColumns, e.columnsNamed(r, accesses))
+}
+
+// columnsNamed returns the columns of accesses that r's names match.
+func (e *omniQuerySpanExtractor) columnsNamed(r *predicateNameResolver, accesses base.SourceColumnSet) base.SourceColumnSet {
+ columns := make(base.SourceColumnSet)
+ for resource := range accesses {
+ rel := e.cat.GetRelation(resource.Schema, resource.Table)
+ if rel == nil {
+ continue
+ }
+ table := strings.ToLower(resource.Table)
+ for _, column := range rel.Columns {
+ name := strings.ToLower(column.Name)
+ if r.wholeRows[table] || r.anyTable[name] || r.ofTable[table][name] {
+ columns[base.ColumnResource{
+ Database: resource.Database,
+ Schema: resource.Schema,
+ Table: resource.Table,
+ Column: column.Name,
+ }] = true
+ }
+ }
+ }
+ return columns
+}
+
+// returningLineage returns, for each column a DML statement's RETURNING list
+// outputs, the columns of accesses it reads, matched by name. It returns nil
+// for a statement without RETURNING.
+func (e *omniQuerySpanExtractor) returningLineage(stmt ast.Node, accesses base.SourceColumnSet) []base.SourceColumnSet {
+ var returning *ast.List
+ var target *ast.RangeVar
+ switch v := stmt.(type) {
+ case *ast.InsertStmt:
+ returning, target = v.ReturningList, v.Relation
+ case *ast.UpdateStmt:
+ returning, target = v.ReturningList, v.Relation
+ case *ast.DeleteStmt:
+ returning, target = v.ReturningList, v.Relation
+ default:
+ }
+ if returning == nil {
+ return nil
+ }
+ var lineage []base.SourceColumnSet
+ for _, item := range returning.Items {
+ rt, ok := item.(*ast.ResTarget)
+ if !ok {
+ continue
+ }
+ if isStarTarget(rt) && target != nil {
+ // RETURNING * outputs every column of the target table in order.
+ for resource := range accesses {
+ if !strings.EqualFold(resource.Table, target.Relname) {
+ continue
+ }
+ if rel := e.cat.GetRelation(resource.Schema, resource.Table); rel != nil {
+ for _, column := range rel.Columns {
+ lineage = append(lineage, base.SourceColumnSet{{
+ Database: resource.Database,
+ Schema: resource.Schema,
+ Table: resource.Table,
+ Column: column.Name,
+ }: true})
+ }
+ }
+ break
+ }
+ continue
+ }
+ r := newPredicateNameResolver(stmt, e.namedFunctionReturnsSet)
+ r.addReferences(rt.Val)
+ lineage = append(lineage, e.columnsNamed(r, accesses))
+ }
+ return lineage
+}
+
+// predicateNameResolver resolves the column references of conditions by name.
+// A reference qualified by a table or its alias matches that table's column;
+// any other reference matches the same-named column of every table, and an
+// output name of a subquery or CTE also reads the columns that produce it.
+type predicateNameResolver struct {
+ // The WHERE (of SELECT, UPDATE, and DELETE), JOIN ON, HAVING, and FILTER
+ // conditions, LIMIT and OFFSET,
+ // arguments of FROM and set-returning functions, USING column names,
+ // NATURAL joins, and INTERSECT and EXCEPT operations of the statement, at
+ // any depth.
+ conditions []ast.Node
+ usingNames []string
+ naturalJoins []*ast.JoinExpr
+ filteringSetOperations []*ast.SelectStmt
+ // projections maps an output name to the expressions that produce it. A
+ // subquery or CTE name maps to its whole query, so a whole-row reference
+ // to it reads every column.
+ projections map[string][]ast.Node
+ // relations maps a name or alias in FROM to the relations it may name; an
+ // alias reused across scopes names several.
+ relations map[string]map[string]bool
+ // derived holds the names of subqueries and CTEs.
+ derived map[string]bool
+
+ anyTable map[string]bool
+ ofTable map[string]map[string]bool
+ wholeRows map[string]bool
+}
+
+func newPredicateNameResolver(stmt ast.Node, returnsSet func(name string) bool) *predicateNameResolver {
+ r := &predicateNameResolver{
+ projections: make(map[string][]ast.Node),
+ relations: make(map[string]map[string]bool),
+ derived: make(map[string]bool),
+ anyTable: make(map[string]bool),
+ ofTable: make(map[string]map[string]bool),
+ wholeRows: make(map[string]bool),
+ }
+ project := func(name string, nodes ...ast.Node) {
+ if name != "" {
+ r.projections[strings.ToLower(name)] = append(r.projections[strings.ToLower(name)], nodes...)
+ }
+ }
+ ast.Inspect(stmt, func(n ast.Node) bool {
+ switch v := n.(type) {
+ case *ast.SelectStmt:
+ r.conditions = append(r.conditions, v.WhereClause, v.HavingClause, v.LimitCount, v.LimitOffset)
+ if v.Op == ast.SETOP_INTERSECT || v.Op == ast.SETOP_EXCEPT {
+ r.filteringSetOperations = append(r.filteringSetOperations, v)
+ }
+ if v.TargetList != nil {
+ for _, item := range v.TargetList.Items {
+ if rt, ok := item.(*ast.ResTarget); ok {
+ name := rt.Name
+ if name == "" {
+ name = figureResTargetName(rt)
+ }
+ project(name, rt.Val)
+ }
+ }
+ }
+ case *ast.UpdateStmt:
+ r.conditions = append(r.conditions, v.WhereClause)
+ case *ast.DeleteStmt:
+ r.conditions = append(r.conditions, v.WhereClause)
+ case *ast.JoinExpr:
+ r.conditions = append(r.conditions, v.Quals)
+ r.usingNames = append(r.usingNames, astStrings(v.UsingClause)...)
+ if v.IsNatural {
+ r.naturalJoins = append(r.naturalJoins, v)
+ }
+ case *ast.FuncCall:
+ r.conditions = append(r.conditions, v.AggFilter)
+ if names := astStrings(v.Funcname); len(names) > 0 && returnsSet(names[len(names)-1]) {
+ r.conditions = append(r.conditions, v.Args)
+ }
+ case *ast.RangeFunction:
+ // A function in FROM returns the rows its arguments ask for.
+ r.conditions = append(r.conditions, v.Functions)
+ case *ast.CommonTableExpr:
+ r.derived[strings.ToLower(v.Ctename)] = true
+ project(v.Ctename, v.Ctequery)
+ for i, name := range astStrings(v.Aliascolnames) {
+ project(name, outputsAt(v.Ctequery, i)...)
+ }
+ case *ast.RangeSubselect:
+ if v.Alias != nil {
+ r.derived[strings.ToLower(v.Alias.Aliasname)] = true
+ project(v.Alias.Aliasname, v.Subquery)
+ for i, name := range astStrings(v.Alias.Colnames) {
+ project(name, outputsAt(v.Subquery, i)...)
+ }
+ }
+ case *ast.RangeVar:
+ relation := strings.ToLower(v.Relname)
+ r.addRelationName(relation, relation)
+ if v.Alias != nil {
+ r.addRelationName(v.Alias.Aliasname, relation)
+ }
+ default:
+ }
+ return true
+ })
+ return r
+}
+
+func (r *predicateNameResolver) addRelationName(name, relation string) {
+ name = strings.ToLower(name)
+ if name == "" {
+ return
+ }
+ if r.relations[name] == nil {
+ r.relations[name] = make(map[string]bool)
+ }
+ r.relations[name][relation] = true
+}
+
+// relationsNamed returns the tables name may refer to, and the subqueries or
+// CTEs it may refer to instead.
+func (r *predicateNameResolver) relationsNamed(name string) (map[string]bool, []string) {
+ name = strings.ToLower(name)
+ tables := make(map[string]bool)
+ var derived []string
+ if r.derived[name] {
+ derived = append(derived, name)
+ }
+ for relation := range r.relations[name] {
+ if r.derived[relation] {
+ derived = append(derived, relation)
+ } else {
+ tables[relation] = true
+ }
+ }
+ return tables, derived
+}
+
+// addSetOperationOutputs adds the outputs of every branch of sel, which an
+// INTERSECT or EXCEPT compares.
+func (r *predicateNameResolver) addSetOperationOutputs(sel *ast.SelectStmt) {
+ if sel == nil {
+ return
+ }
+ if sel.Op != ast.SETOP_NONE {
+ r.addSetOperationOutputs(sel.Larg)
+ r.addSetOperationOutputs(sel.Rarg)
+ return
+ }
+ if sel.TargetList == nil {
+ return
+ }
+ for _, item := range sel.TargetList.Items {
+ rt, ok := item.(*ast.ResTarget)
+ if !ok {
+ continue
+ }
+ if !isStarTarget(rt) {
+ r.addReferences(rt.Val)
+ continue
+ }
+ if sel.FromClause != nil {
+ for _, from := range sel.FromClause.Items {
+ r.addJoinedRelation(from)
+ }
+ }
+ }
+}
+
+func (r *predicateNameResolver) addJoinedRelation(node ast.Node) {
+ switch v := node.(type) {
+ case *ast.RangeVar:
+ if v.Alias != nil && v.Alias.Aliasname != "" {
+ r.addWholeRow(v.Alias.Aliasname)
+ } else {
+ r.addWholeRow(v.Relname)
+ }
+ case *ast.RangeSubselect:
+ r.addReferences(v.Subquery)
+ case *ast.JoinExpr:
+ r.addJoinedRelation(v.Larg)
+ r.addJoinedRelation(v.Rarg)
+ default:
+ }
+}
+
+func (r *predicateNameResolver) addReferences(node ast.Node) {
+ ast.Inspect(node, func(n ast.Node) bool {
+ if ref, ok := n.(*ast.ColumnRef); ok {
+ r.addReference(ref)
+ }
+ return true
+ })
+}
+
+func (r *predicateNameResolver) addReference(ref *ast.ColumnRef) {
+ if ref.Fields == nil {
+ return
+ }
+ var fields []string
+ star := false
+ for _, field := range ref.Fields.Items {
+ switch v := field.(type) {
+ case *ast.String:
+ fields = append(fields, v.Str)
+ case *ast.A_Star:
+ star = true
+ default:
+ }
+ }
+ if len(fields) == 0 {
+ return
+ }
+ last := fields[len(fields)-1]
+ // t.* reads every column of t; a bare t may be a column or a whole-row
+ // reference.
+ if star || len(fields) == 1 {
+ r.addWholeRow(last)
+ return
+ }
+ tables, derived := r.relationsNamed(fields[len(fields)-2])
+ for table := range tables {
+ if r.ofTable[table] == nil {
+ r.ofTable[table] = make(map[string]bool)
+ }
+ r.ofTable[table][strings.ToLower(last)] = true
+ }
+ if len(derived) > 0 || len(tables) == 0 {
+ r.addName(last)
+ }
+}
+
+func (r *predicateNameResolver) addWholeRow(name string) {
+ tables, derived := r.relationsNamed(name)
+ for table := range tables {
+ r.wholeRows[table] = true
+ }
+ for _, relation := range derived {
+ r.addName(relation)
+ }
+ r.addName(name)
+}
+
+func (r *predicateNameResolver) addName(name string) {
+ name = strings.ToLower(name)
+ if name == "" || r.anyTable[name] {
+ return
+ }
+ r.anyTable[name] = true
+ for _, projection := range r.projections[name] {
+ r.addReferences(projection)
+ }
+}
+
+// outputsAt returns the expressions that produce the i-th output column of
+// query, or query itself when they cannot be told apart.
+func outputsAt(query ast.Node, i int) []ast.Node {
+ sel, ok := query.(*ast.SelectStmt)
+ if !ok || sel == nil {
+ return []ast.Node{query}
+ }
+ if sel.Op != ast.SETOP_NONE {
+ return append(outputsAt(sel.Larg, i), outputsAt(sel.Rarg, i)...)
+ }
+ if sel.TargetList == nil || i >= len(sel.TargetList.Items) {
+ return []ast.Node{query}
+ }
+ for _, item := range sel.TargetList.Items[:i+1] {
+ if rt, ok := item.(*ast.ResTarget); !ok || isStarTarget(rt) {
+ return []ast.Node{query}
+ }
+ }
+ return []ast.Node{sel.TargetList.Items[i].(*ast.ResTarget).Val}
+}
+
+func astStrings(list *ast.List) []string {
+ if list == nil {
+ return nil
+ }
+ var result []string
+ for _, item := range list.Items {
+ if s, ok := item.(*ast.String); ok {
+ result = append(result, s.Str)
+ }
+ }
+ return result
+}
diff --git a/backend/plugin/parser/pg/query_span_predicate_test.go b/backend/plugin/parser/pg/query_span_predicate_test.go
new file mode 100644
index 0000000000..00e659b2e6
--- /dev/null
+++ b/backend/plugin/parser/pg/query_span_predicate_test.go
@@ -0,0 +1,294 @@
+package pg
+
+import (
+ "context"
+ "slices"
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/bytebase/omni/pg/ast"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func predicateColumnNames(columns base.SourceColumnSet) []string {
+ names := []string{}
+ for column := range columns {
+ if column.Column != "" {
+ names = append(names, column.Table+"."+column.Column)
+ }
+ }
+ slices.Sort(names)
+ return names
+}
+
+func TestGetQuerySpanPredicateColumns(t *testing.T) {
+ testCases := []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"where", "SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},
+ {"where function", "SELECT id FROM t WHERE ascii(substr(ssn, 1, 1)) > 70", []string{"t.ssn"}},
+ {"row comparison", "SELECT id FROM t WHERE (ssn, id) > ('1', 0)", []string{"t.id", "t.ssn"}},
+ {"join on", "SELECT o.amt FROM o JOIN t ON t.id = o.id AND t.ssn = '1'", []string{"o.id", "t.id", "t.ssn"}},
+ {"left join on", "SELECT o.amt FROM o LEFT JOIN t ON t.id = o.id AND t.ssn = '1'", []string{"o.id", "t.id", "t.ssn"}},
+ {"join using", "SELECT a.id FROM t a JOIN t b USING (ssn)", []string{"t.ssn"}},
+ {"having", "SELECT id FROM t GROUP BY id HAVING max(ssn) > '1'", []string{"t.ssn"}},
+ {"in subquery", "SELECT amt FROM o WHERE id IN (SELECT id FROM t WHERE ssn = '1')", []string{"o.id", "t.id", "t.ssn"}},
+ {"subquery output", "SELECT amt FROM o WHERE '1' = (SELECT max(ssn) FROM t)", []string{"t.ssn"}},
+ {"correlated exists", "SELECT amt FROM o WHERE EXISTS (SELECT 1 FROM t WHERE t.id = o.id AND t.ssn = '1')", []string{"o.id", "t.id", "t.ssn"}},
+ {"subquery in select list", "SELECT (SELECT count(*) FROM t WHERE ssn = '1') AS c", []string{"t.ssn"}},
+ {"derived table", "SELECT id FROM (SELECT * FROM t WHERE ssn = '1') s", []string{"t.ssn"}},
+ {"derived table output", "SELECT s.id FROM (SELECT id, ssn AS x FROM t) s WHERE s.x = '1'", []string{"t.ssn"}},
+ {"lateral subquery", "SELECT o.amt FROM o, LATERAL (SELECT 1 FROM t WHERE t.id = o.id AND t.ssn = '1') l", []string{"o.id", "t.id", "t.ssn"}},
+ {"cte", "WITH c AS (SELECT id FROM t WHERE ssn = '1') SELECT id FROM c", []string{"t.ssn"}},
+ {"cte output", "WITH c AS (SELECT id, ssn FROM t) SELECT id FROM c WHERE ssn = '1'", []string{"t.ssn"}},
+ {"set operation arm", "SELECT id FROM t WHERE ssn = '1' UNION SELECT id FROM o", []string{"t.ssn"}},
+ {"except filters by the other branch", "SELECT 'guess' AS x EXCEPT SELECT ssn FROM t", []string{"t.ssn"}},
+ {"intersect filters by the other branch", "SELECT 'guess' AS x INTERSECT ALL SELECT ssn FROM t", []string{"t.ssn"}},
+ {"union filters nothing", "SELECT 'guess' AS x UNION SELECT ssn FROM t", []string{}},
+ {"explain analyze", "EXPLAIN ANALYZE SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},
+ {"aggregate filter", "SELECT sum(id) FILTER (WHERE ssn = '1') AS x FROM t", []string{"t.ssn"}},
+ {"natural join", "SELECT 1 FROM t NATURAL JOIN (SELECT '1'::text AS ssn) x", []string{"t.ssn"}},
+ {"full join using", "SELECT id FROM t FULL JOIN o USING (id) WHERE id > 0", []string{"o.id", "t.id"}},
+ {"limit subquery", "SELECT id FROM t LIMIT (SELECT count(*) FROM t WHERE ssn = '1')", []string{"t.ssn"}},
+ {"limit value", "SELECT id FROM t LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},
+ {"set operation limit", "SELECT 1 AS x UNION ALL SELECT 2 LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},
+ {"set-returning function in from", "SELECT count(*) AS c FROM t CROSS JOIN LATERAL generate_series(1, ascii(substr(t.ssn, 1, 1)))", []string{"t.ssn"}},
+ {"set-returning function in select list", "SELECT generate_series(1, ascii(substr(ssn, 1, 1))) AS g FROM t", []string{"t.ssn"}},
+ {"scalar function in select list", "SELECT upper(ssn) AS u FROM t", []string{}},
+ {"unknown function", "SELECT id FROM t WHERE levenshtein(ssn, 'a') > 0", []string{"t.ssn"}},
+ {"unknown function in from", "SELECT t.id FROM t, LATERAL unknown_rows(t.ssn) f WHERE f.a = 1", []string{"t.ssn"}},
+ {"select list only", "SELECT ssn FROM t", []string{}},
+ {"condition in select list", "SELECT CASE WHEN ssn = '1' THEN 1 END AS x FROM t", []string{}},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ span := spanFor(t, tc.statement, healthySchema())
+ require.Equal(t, tc.want, predicateColumnNames(span.PredicateColumns))
+ })
+ }
+}
+
+func TestGetQuerySpanPredicateColumnsOfFunctionBodies(t *testing.T) {
+ schema := healthySchema()
+ // * over an unknown function fails analysis of the body that uses it.
+ unanalyzable := "(SELECT * FROM o CROSS JOIN unknown_rows() u) o JOIN t ON t.id = o.id AND t.ssn = guess"
+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{
+ {Name: "probe", Definition: "CREATE FUNCTION public.probe(guess text) RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT t.id FROM t WHERE t.ssn = guess; END; $$"},
+ {Name: "probe_sql", Definition: "CREATE FUNCTION public.probe_sql(guess text) RETURNS SETOF int LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = guess $$"},
+ {Name: "probe_fixed", Definition: "CREATE FUNCTION public.probe_fixed() RETURNS SETOF int LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = 'guess' $$"},
+ {Name: "probe_multi", Definition: "CREATE FUNCTION public.probe_multi() RETURNS int LANGUAGE sql AS $$ SELECT pg_sleep(0) FROM t WHERE ssn = 'guess'; SELECT 1 $$"},
+ {Name: "probe_perform", Definition: "CREATE FUNCTION public.probe_perform() RETURNS int LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(0) FROM t WHERE ssn = 'guess'; RETURN 1; END; $$"},
+ {Name: "probe_if", Definition: "CREATE FUNCTION public.probe_if() RETURNS int LANGUAGE plpgsql AS $$ BEGIN IF (SELECT ssn FROM t LIMIT 1) = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},
+ {Name: "probe_variable", Definition: "CREATE FUNCTION public.probe_variable() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t WHERE id = 1; IF v = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},
+ {Name: "probe_update", Definition: "CREATE FUNCTION public.probe_update() RETURNS int LANGUAGE sql AS $$ UPDATE t SET id = id WHERE ssn = 'guess'; SELECT 1 $$"},
+ {Name: "probe_exec", Definition: "CREATE FUNCTION public.probe_exec() RETURNS int LANGUAGE plpgsql AS $$ BEGIN DELETE FROM t WHERE ssn = 'guess'; RETURN 1; END; $$"},
+ {Name: "probe_table", Definition: "CREATE FUNCTION public.probe_table() RETURNS TABLE(id int) LANGUAGE sql AS $$ SELECT id FROM t WHERE ssn = 'guess' $$"},
+ {Name: "probe_record", Definition: "CREATE FUNCTION public.probe_record() RETURNS int LANGUAGE plpgsql AS $$ DECLARE r record; BEGIN SELECT ssn INTO r FROM t LIMIT 1; IF r.ssn = 'guess' THEN RETURN 1; END IF; RETURN 0; END; $$"},
+ {Name: "probe_execute", Definition: "CREATE FUNCTION public.probe_execute() RETURNS int LANGUAGE plpgsql AS $$ BEGIN EXECUTE 'SELECT pg_sleep(0) FROM t ' || 'WHERE ssn = ''guess'''; RETURN 1; END; $$"},
+ {Name: "probe_count", Definition: "CREATE FUNCTION public.probe_count() RETURNS int LANGUAGE sql AS $$ SELECT count(*)::int FROM t WHERE ssn = 'guess' $$"},
+ {Name: "probe_bool", Definition: "CREATE FUNCTION public.probe_bool() RETURNS boolean LANGUAGE sql AS $$ SELECT EXISTS (SELECT 1 FROM t WHERE ssn = 'guess') $$"},
+ {Name: "probe_raise", Definition: "CREATE FUNCTION public.probe_raise() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t LIMIT 1; RAISE EXCEPTION '%', v; END; $$"},
+ // The overload that reads ssn loads first, so a body chosen by name alone
+ // would be the harmless one.
+ {Name: "probe_over", Definition: "CREATE FUNCTION public.probe_over(x text) RETURNS boolean LANGUAGE sql AS $$ SELECT EXISTS (SELECT 1 FROM t WHERE ssn = x) $$"},
+ {Name: "probe_over", Definition: "CREATE FUNCTION public.probe_over(x int) RETURNS boolean LANGUAGE sql AS $$ SELECT true $$"},
+ // A missing return type makes the catalog install these as stand-ins.
+ {Name: "probe_stand_in", Definition: "CREATE FUNCTION public.probe_stand_in(x text) RETURNS SETOF missing_tbl LANGUAGE sql AS $$ SELECT * FROM t WHERE ssn = x $$"},
+ {Name: "probe_stand_in", Definition: "CREATE FUNCTION public.probe_stand_in(x int) RETURNS SETOF missing_tbl LANGUAGE sql AS $$ SELECT * FROM o $$"},
+ {Name: "probe_stand_in_pl", Definition: "CREATE FUNCTION public.probe_stand_in_pl(x text) RETURNS SETOF missing_tbl LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT * FROM t WHERE ssn = x; END; $$"},
+ {Name: "probe_discard", Definition: "CREATE FUNCTION public.probe_discard() RETURNS int LANGUAGE sql AS $$ SELECT pg_sleep(CASE WHEN ssn = 'guess' THEN 0 ELSE 0 END) FROM t; SELECT 1 $$"},
+ {Name: "probe_declare", Definition: "CREATE FUNCTION public.probe_declare() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text := (SELECT ssn FROM t LIMIT 1); BEGIN RAISE EXCEPTION '%', v; END; $$"},
+ {Name: "proc_check", Definition: "CREATE PROCEDURE public.proc_check(x text) LANGUAGE plpgsql AS $$ BEGIN IF x = 'guess' THEN RAISE EXCEPTION 'hit'; END IF; END; $$"},
+ {Name: "probe_call_subquery", Definition: "CREATE FUNCTION public.probe_call_subquery() RETURNS int LANGUAGE plpgsql AS $$ BEGIN CALL proc_check((SELECT ssn FROM t LIMIT 1)); RETURN 1; END; $$"},
+ {Name: "probe_call", Definition: "CREATE FUNCTION public.probe_call() RETURNS int LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN SELECT ssn INTO v FROM t LIMIT 1; CALL proc_check(v); RETURN 1; END; $$"},
+ // The last SELECT reads a table the snapshot lacks, so the body fails analysis.
+ {Name: "probe_failing", Definition: "CREATE FUNCTION public.probe_failing() RETURNS SETOF record LANGUAGE sql AS $$ SELECT pg_sleep(0) FROM t WHERE ssn = 'guess'; SELECT * FROM missing_tbl $$"},
+ {Name: "probe_for", Definition: "CREATE FUNCTION public.probe_for() RETURNS SETOF int LANGUAGE plpgsql AS $$ BEGIN FOR i IN 1..(SELECT ascii(ssn) FROM t LIMIT 1) LOOP RETURN NEXT i; END LOOP; END; $$"},
+ {Name: "probe_join", Definition: "CREATE FUNCTION public.probe_join(guess text) RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT o.id FROM " + unanalyzable + "; END; $$"},
+ {Name: "probe_join_sql", Definition: "CREATE FUNCTION public.probe_join_sql(guess text) RETURNS SETOF int LANGUAGE sql AS $$ SELECT o.id FROM " + unanalyzable + " $$"},
+ }
+ testCases := []struct {
+ statement string
+ want []string
+ }{
+ {"SELECT count(*) AS c FROM probe('guess')", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_sql('guess')", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_fixed()", []string{"t.ssn"}},
+ {"SELECT probe_multi() AS x", []string{"t.ssn"}},
+ {"SELECT probe_perform() AS x", []string{"t.ssn"}},
+ {"SELECT probe_if() AS x", []string{"t.ssn"}},
+ {"SELECT probe_variable() AS x", []string{"t.id", "t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_for()", []string{"t.ssn"}},
+ {"SELECT probe_update() AS x", []string{"t.ssn"}},
+ {"SELECT probe_record() AS x", []string{"t.ssn"}},
+ {"SELECT probe_raise() AS x", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_stand_in('guess')", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_stand_in_pl('guess')", []string{"t.ssn"}},
+ {"SELECT probe_discard() AS x", []string{"t.ssn"}},
+ {"SELECT probe_declare() AS x", []string{"t.ssn"}},
+ {"SELECT probe_call() AS x", []string{"t.ssn"}},
+ {"SELECT probe_call_subquery() AS x", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_failing() AS f(a int)", []string{"t.ssn"}},
+ {"SELECT probe_over('guess') AS b", []string{"t.ssn"}},
+ {"SELECT probe_over(1) AS b", []string{}},
+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE probe_over(1)", []string{"t.ssn"}},
+ // omni leaves a set operation's LIMIT out of its tree.
+ {"SELECT 1 AS x UNION ALL SELECT 2 LIMIT probe_count()", []string{"t.ssn"}},
+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE probe_bool()", []string{"t.ssn"}},
+ {"SELECT * FROM o CROSS JOIN unknown_rows() u WHERE public.probe_bool()", []string{"t.ssn"}},
+ {"SELECT probe_execute() AS x", []string{"t.ssn"}},
+ {"SELECT probe_exec() AS x", []string{"t.ssn"}},
+ // * over an unknown function sends the outer query to the fallback.
+ {"SELECT * FROM probe_table() p CROSS JOIN unknown_rows() u", []string{"t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_join('guess')", []string{"o.id", "t.id", "t.ssn"}},
+ {"SELECT count(*) AS c FROM probe_join_sql('guess')", []string{"o.id", "t.id", "t.ssn"}},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.statement, func(t *testing.T) {
+ span := spanFor(t, tc.statement, schema)
+ require.Equal(t, tc.want, predicateColumnNames(span.PredicateColumns))
+ })
+ }
+}
+
+func TestGetQuerySpanResultLineage(t *testing.T) {
+ testCases := []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"set operation in subquery", "SELECT s.a FROM (SELECT ssn AS a FROM t UNION SELECT email FROM t) s", []string{"t.email", "t.ssn"}},
+ {"set operation in cte", "WITH c AS (SELECT ssn FROM t UNION SELECT email FROM t) SELECT ssn FROM c", []string{"t.email", "t.ssn"}},
+ {"except returns left branch rows", "SELECT s.a FROM (SELECT ssn AS a FROM t EXCEPT SELECT email FROM t) s", []string{"t.ssn"}},
+ {"window before column in subquery", "SELECT s.x FROM (SELECT row_number() OVER (ORDER BY ssn) AS rn, email AS x FROM t) s", []string{"t.email"}},
+ {"full join using column", "SELECT id FROM t FULL JOIN o USING (id)", []string{"o.id", "t.id"}},
+ {"aggregate order by", "SELECT string_agg(email, ',' ORDER BY ssn) AS x FROM t", []string{"t.email", "t.ssn"}},
+ {"unknown function in from", "SELECT f.a FROM t CROSS JOIN LATERAL unknown_rows(t.ssn) f", []string{"t.ssn"}},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ span := spanFor(t, tc.statement, healthySchema())
+ require.Len(t, span.Results, 1)
+ require.Equal(t, tc.want, predicateColumnNames(span.Results[0].SourceColumns))
+ })
+ }
+}
+
+func TestGetQuerySpanResultLineageOfFunctionBodies(t *testing.T) {
+ schema := healthySchema()
+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{
+ {Name: "read_cursor", Definition: "CREATE FUNCTION public.read_cursor() RETURNS text LANGUAGE plpgsql AS $$ DECLARE c CURSOR FOR SELECT ssn FROM t; v text; BEGIN OPEN c; FETCH c INTO v; RETURN v; END; $$"},
+ {Name: "loop_cursor", Definition: "CREATE FUNCTION public.loop_cursor() RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE c CURSOR FOR SELECT ssn FROM t; BEGIN FOR r IN c LOOP RETURN NEXT r.ssn; END LOOP; END; $$"},
+ {Name: "returning_sql", Definition: "CREATE FUNCTION public.returning_sql() RETURNS SETOF text LANGUAGE sql AS $$ UPDATE t SET id = id RETURNING ssn $$"},
+ {Name: "returning_table", Definition: "CREATE FUNCTION public.returning_table() RETURNS TABLE(i int, e text, s text) LANGUAGE sql AS $$ DELETE FROM t WHERE false RETURNING * $$"},
+ {Name: "returning_pl", Definition: "CREATE FUNCTION public.returning_pl() RETURNS text LANGUAGE plpgsql AS $$ DECLARE v text; BEGIN UPDATE t SET id = id WHERE id = 1 RETURNING ssn INTO v; RETURN v; END; $$"},
+ }
+ testCases := []struct {
+ statement string
+ want []string
+ }{
+ {"SELECT read_cursor() AS x", []string{"t.ssn"}},
+ {"SELECT loop_cursor() AS x", []string{"t.ssn"}},
+ {"SELECT returning_sql() AS x", []string{"t.ssn"}},
+ {"SELECT returning_pl() AS x", []string{"t.ssn"}},
+ {"SELECT r.s FROM returning_table() r", []string{"t.ssn"}},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.statement, func(t *testing.T) {
+ span := spanFor(t, tc.statement, schema)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, tc.want, predicateColumnNames(span.Results[0].SourceColumns))
+ })
+ }
+}
+
+func TestGetQuerySpanKeepsSQLBodyAccesses(t *testing.T) {
+ schema := degradedSchema()
+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{
+ {Name: "probe_sql", Definition: "CREATE FUNCTION public.probe_sql() RETURNS int LANGUAGE sql AS $$ SELECT 1 FROM t WHERE secret = 'guess' $$"},
+ }
+ span := spanFor(t, "SELECT probe_sql() AS x", schema)
+ require.Contains(t, span.SourceColumns, base.ColumnResource{Database: "db", Schema: "public", Table: "t"})
+ require.NotNil(t, span.UnresolvedColumnsError, "t, read only in the body's condition, was synced with no columns")
+}
+
+func TestGetQuerySpanFallbackKeepsFunctionAccesses(t *testing.T) {
+ schema := degradedSchema()
+ schema.Schemas[0].Functions = []*metadatapb.FunctionMetadata{
+ {Name: "probe_t", Definition: "CREATE FUNCTION public.probe_t() RETURNS TABLE(id int) LANGUAGE plpgsql AS $$ BEGIN RETURN QUERY SELECT 1 FROM t WHERE secret = 'guess'; END; $$"},
+ }
+ // * over an unknown function sends the statement to the fallback.
+ span := spanFor(t, "SELECT * FROM probe_t() p CROSS JOIN unknown_rows() u", schema)
+ require.Contains(t, span.SourceColumns, base.ColumnResource{Database: "db", Schema: "public", Table: "t"})
+ require.NotNil(t, span.UnresolvedColumnsError, "t, read only inside the function, was synced with no columns")
+}
+
+func newFallbackTestExtractor(t *testing.T) *omniQuerySpanExtractor {
+ t.Helper()
+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{healthySchema()})
+ e := newOmniQuerySpanExtractor("db", nil, base.GetQuerySpanContext{GetDatabaseMetadataFunc: getter, ListDatabaseNamesFunc: lister})
+ e.ctx = context.Background()
+ require.NoError(t, e.initCatalog())
+ return e
+}
+
+func TestCollectFallbackPredicateColumns(t *testing.T) {
+ testCases := []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"where", "SELECT id FROM t WHERE ssn = '1'", []string{"t.ssn"}},
+ {"qualified reference matches its table", "SELECT o.amt FROM o JOIN t ON o.id = t.id WHERE o.amt > 0", []string{"o.amt", "o.id", "t.id"}},
+ {"unqualified reference matches every table", "SELECT 1 FROM o JOIN t USING (id) WHERE id > 0", []string{"o.id", "t.id"}},
+ {"reused alias matches every table it names", "SELECT 1 FROM t x WHERE EXISTS (SELECT 1 FROM o x WHERE x.id = 1)", []string{"o.id", "t.id"}},
+ {"subquery in a condition reads every name in it", "SELECT 1 FROM o WHERE EXISTS (WITH c AS (SELECT id FROM t WHERE ssn = '1') SELECT 1 FROM c)", []string{"o.id", "t.id", "t.ssn"}},
+ {"cte column alias", "WITH c(a, b) AS (SELECT id, ssn FROM t) SELECT a FROM c WHERE b = '1'", []string{"t.ssn"}},
+ {"recursive cte column alias", "WITH RECURSIVE c(a, n) AS (SELECT ssn, 1 FROM t UNION ALL SELECT a, n + 1 FROM c WHERE n < 3) SELECT a FROM c", []string{}},
+ {"derived column alias", "SELECT 1 FROM (SELECT id, ssn FROM t) s(a, b) WHERE s.b = '1'", []string{"t.ssn"}},
+ {"output name", "SELECT 1 FROM (SELECT ssn AS x FROM t) s WHERE x = '1'", []string{"t.ssn"}},
+ {"whole row", "SELECT 1 FROM t WHERE t::text LIKE '%1%'", []string{"t.email", "t.id", "t.ssn"}},
+ {"aliased whole row", "SELECT 1 FROM t x WHERE ROW(x.*) IS NOT NULL", []string{"t.email", "t.id", "t.ssn"}},
+ {"join using", "SELECT 1 FROM t a JOIN t b USING (ssn)", []string{"t.ssn"}},
+ {"having", "SELECT id FROM t GROUP BY id HAVING max(ssn) > '1'", []string{"t.ssn"}},
+ {"aggregate filter", "SELECT count(*) FILTER (WHERE ssn = '1') FROM t", []string{"t.ssn"}},
+ {"natural join reads both sides", "SELECT 1 FROM o NATURAL JOIN (SELECT id FROM t) x", []string{"o.amt", "o.id", "t.id"}},
+ {"select list only", "SELECT ssn FROM t", []string{}},
+ {"except filters by the other branch", "SELECT 'guess' EXCEPT SELECT ssn FROM t", []string{"t.ssn"}},
+ {"intersect over star", "SELECT 1, 'a', 'b' INTERSECT SELECT * FROM t", []string{"t.email", "t.id", "t.ssn"}},
+ {"union filters nothing", "SELECT 'guess' UNION SELECT ssn FROM t", []string{}},
+ {"limit value", "SELECT * FROM t LIMIT (SELECT CASE WHEN ssn = 'guess' THEN 1 ELSE 0 END FROM t LIMIT 1)", []string{"t.ssn"}},
+ {"function in from", "SELECT * FROM t CROSS JOIN LATERAL generate_series(1, ascii(substr(t.ssn, 1, 1))) g", []string{"t.ssn"}},
+ {"set-returning function in select list", "SELECT generate_series(1, ascii(substr(ssn, 1, 1))) FROM t", []string{"t.ssn"}},
+ {"scalar function in select list", "SELECT upper(ssn) FROM t", []string{}},
+ }
+ accesses := base.SourceColumnSet{
+ {Database: "db", Schema: "public", Table: "t"}: true,
+ {Database: "db", Schema: "public", Table: "o"}: true,
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ e := newFallbackTestExtractor(t)
+ stmts, err := ParsePg(tc.statement)
+ require.NoError(t, err)
+ e.collectFallbackPredicateColumns(stmts[0].AST, accesses)
+ require.Equal(t, tc.want, predicateColumnNames(e.predicateColumns))
+ })
+ }
+}
+
+func TestGetQuerySpanFallbackKeepsPredicateColumns(t *testing.T) {
+ // * over the unknown columns of a function fails omni's analysis.
+ statement := "SELECT * FROM o JOIN t ON t.id = o.id AND t.ssn = '1' CROSS JOIN unknown_rows() f"
+ stmts, err := ParsePg(statement)
+ require.NoError(t, err)
+ _, err = newFallbackTestExtractor(t).analyzeSelect(stmts[0].AST.(*ast.SelectStmt))
+ require.Error(t, err, "the statement must fail analysis to reach the fallback")
+
+ span := spanFor(t, statement, healthySchema())
+ require.Equal(t, []string{"o.id", "t.id", "t.ssn"}, predicateColumnNames(span.PredicateColumns))
+}
diff --git a/backend/plugin/parser/pg/query_span_test.go b/backend/plugin/parser/pg/query_span_test.go
index cdfde7d715..945078f859 100644
--- a/backend/plugin/parser/pg/query_span_test.go
+++ b/backend/plugin/parser/pg/query_span_test.go
@@ -632,16 +632,12 @@ func TestIsUltimatelyPlainColumnStopsOnCyclicSubquery(t *testing.T) {
})
}
-func TestCollectQueryPredicateColumnsStopsOnCyclicQuery(t *testing.T) {
+func TestCollectPredicateColumnsStopsOnCyclicQuery(t *testing.T) {
mustNotOverflow(t, func() {
q := &catalog.Query{}
q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}
extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})
- analyzer := &plpgsqlAnalyzer{
- extractor: extractor,
- scope: newVariableScope(nil),
- }
- analyzer.collectQueryPredicateColumns(q)
+ extractor.collectPredicateColumns(q)
})
}
diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
index 28adc2e193..d76b25483b 100644
--- a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
@@ -90,6 +90,7 @@ func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {
"SELECT email FROM public.t",
"SELECT t.email FROM public.t",
"SELECT * FROM public.t WHERE email = 'x'",
+ "EXPLAIN ANALYZE SELECT * FROM public.t WHERE email = 'x'",
"SELECT id, email FROM public.t ORDER BY ssn",
"WITH c AS (SELECT * FROM public.t) SELECT * FROM c",
// A non-recursive CTE cannot reference itself; the inner t is the physical table.
diff --git a/backend/plugin/parser/pg/test-data/query_span.yaml b/backend/plugin/parser/pg/test-data/query_span.yaml
index dcec60021e..62a7209f74 100644
--- a/backend/plugin/parser/pg/test-data/query_span.yaml
+++ b/backend/plugin/parser/pg/test-data/query_span.yaml
@@ -855,6 +855,21 @@
table: task_run
column: ""
predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: issue
+ column: id
+ - server: ""
+ database: db
+ schema: public
+ table: issue
+ column: payload
+ - server: ""
+ database: db
+ schema: public
+ table: issue
+ column: pipeline_id
- server: ""
database: db
schema: public
@@ -865,11 +880,41 @@
schema: public
table: principal
column: email
+ - server: ""
+ database: db
+ schema: public
+ table: principal
+ column: id
+ - server: ""
+ database: db
+ schema: public
+ table: task
+ column: id
+ - server: ""
+ database: db
+ schema: public
+ table: task
+ column: pipeline_id
+ - server: ""
+ database: db
+ schema: public
+ table: task
+ column: status
- server: ""
database: db
schema: public
table: task_run
column: creator_id
+ - server: ""
+ database: db
+ schema: public
+ table: task_run
+ column: status
+ - server: ""
+ database: db
+ schema: public
+ table: task_run
+ column: task_id
- description: Test For PLpgSQL User Defined Function in expression
statement: SELECT plpgsql_t(1);
defaultDatabase: db
@@ -1114,7 +1159,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: to_jsonb accept table as argument with from table alias
statement: SELECT to_jsonb(t1) FROM t t1;
defaultDatabase: db
@@ -1390,7 +1440,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Test For Complex PLpgSQL Function As Table Source
statement: SELECT * FROM plpgsql_complex_t(1);
defaultDatabase: db
@@ -1458,7 +1513,17 @@
schema: public
table: t2
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
+ - server: ""
+ database: db
+ schema: public
+ table: t2
+ column: a
- description: Test For PLpgSQL Function As Table Source
statement: SELECT * FROM plpgsql_t(1);
defaultDatabase: db
@@ -1507,7 +1572,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Test For Cascade Function As Table Source
statement: SELECT * FROM cascade_t(1);
defaultDatabase: db
@@ -1560,7 +1630,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Test For Function As Table Source
statement: SELECT * FROM select_t(1);
defaultDatabase: db
@@ -1609,7 +1684,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Test For Explain statements
statement: EXPLAIN SELECT 1;
defaultDatabase: db
@@ -2060,7 +2140,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Correlated subquery in the target field clause.
statement: SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total_paintings FROM galleries g;
defaultDatabase: db
@@ -2478,7 +2563,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: Simple CTE
statement: WITH t1 AS (SELECT * FROM t) SELECT * FROM t1;
defaultDatabase: db
@@ -3026,7 +3116,17 @@
schema: public
table: users
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: payments
+ column: user_id
+ - server: ""
+ database: db
+ schema: public
+ table: users
+ column: id
- description: jsonb_array_elements as table source
statement: SELECT * FROM t, jsonb_array_elements(a) AS elems WHERE id = 1;
defaultDatabase: db
@@ -3079,7 +3179,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: jsonb_array_elements with column alias
statement: SELECT * FROM t, jsonb_array_elements(a) AS elems(val) WHERE id = 1;
defaultDatabase: db
@@ -3132,7 +3237,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: jsonb_array_elements_text as table source with alias
statement: SELECT * FROM t CROSS JOIN LATERAL jsonb_array_elements_text(a) AS part WHERE id = 1;
defaultDatabase: db
@@ -3185,7 +3295,12 @@
schema: public
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: public
+ table: t
+ column: a
- description: array subscript in select list
statement: |-
SELECT
diff --git a/backend/tests/sensitive_data_test.go b/backend/tests/sensitive_data_test.go
index aa3c9577d4..e2714d7067 100644
--- a/backend/tests/sensitive_data_test.go
+++ b/backend/tests/sensitive_data_test.go
@@ -6,12 +6,15 @@ import (
"fmt"
"strings"
"testing"
+ "time"
"connectrpc.com/connect"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
+ "google.golang.org/genproto/googleapis/type/expr"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/durationpb"
+ "google.golang.org/protobuf/types/known/fieldmaskpb"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
@@ -92,6 +95,7 @@ func TestSensitiveData(t *testing.T) {
name varchar(220),
author varchar(220)
);
+ CREATE TABLE ghost();
`
insertData = `
INSERT INTO tech_book VALUES
@@ -285,7 +289,7 @@ func TestSensitiveData(t *testing.T) {
// but the masking pipeline should redact it.
errorLeakResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
Name: database.Name,
- Statement: "SELECT author::uuid FROM tech_book WHERE id = 1",
+ Statement: "SELECT author::uuid FROM tech_book WHERE name = 'bytebase'",
}))
a.NoError(err)
a.Equal(1, len(errorLeakResp.Msg.Results))
@@ -297,7 +301,7 @@ func TestSensitiveData(t *testing.T) {
// Same query on a non-masked column should preserve the original error.
nonMaskedErrorResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
Name: database.Name,
- Statement: "SELECT name::uuid FROM tech_book WHERE id = 1",
+ Statement: "SELECT name::uuid FROM tech_book WHERE name = 'bytebase'",
}))
a.NoError(err)
a.Equal(1, len(nonMaskedErrorResp.Msg.Results))
@@ -305,6 +309,64 @@ func TestSensitiveData(t *testing.T) {
// The error should NOT be redacted — name is not masked.
a.NotContains(nonMaskedErrorResp.Msg.Results[0].Error, "data masking policies")
+ const refusedPredicate = "Using sensitive columns in query conditions is not allowed"
+ for _, statement := range []string{
+ "SELECT name FROM tech_book WHERE author = 'bber'",
+ "SELECT b.name FROM tech_book b JOIN tech_book a ON a.id = b.id AND ascii(substr(a.author, 1, 1)) > 70",
+ "SELECT name FROM tech_book WHERE EXISTS (SELECT 1 FROM tech_book t WHERE t.author LIKE 'E%')",
+ "EXPLAIN ANALYZE SELECT name FROM tech_book WHERE author LIKE 'E%'",
+ "SELECT count(*) FILTER (WHERE author LIKE 'E%') FROM tech_book",
+ "SELECT count(*) FROM tech_book NATURAL JOIN (SELECT 'bber'::varchar AS author) x",
+ // Running this would fail on the cast, so the refusal proves it never ran.
+ "SELECT name FROM tech_book WHERE author::int = 1",
+ } {
+ resp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
+ Name: database.Name,
+ Statement: statement,
+ }))
+ a.NoError(err, statement)
+ a.Len(resp.Msg.Results, 1, statement)
+ a.Contains(resp.Msg.Results[0].Error, refusedPredicate, statement)
+ a.Empty(resp.Msg.Results[0].Rows, statement)
+ }
+
+ unmaskedPredicateResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
+ Name: database.Name,
+ Statement: "SELECT name FROM tech_book WHERE name = 'bytebase'",
+ }))
+ a.NoError(err)
+ a.Len(unmaskedPredicateResp.Msg.Results, 1)
+ a.Empty(unmaskedPredicateResp.Msg.Results[0].Error)
+ a.Len(unmaskedPredicateResp.Msg.Results[0].Rows, 1)
+
+ for _, statement := range []string{
+ "SELECT a FROM (SELECT author AS a FROM tech_book UNION ALL SELECT name FROM tech_book) s",
+ "WITH c AS (SELECT author FROM tech_book UNION SELECT name FROM tech_book) SELECT author FROM c",
+ "SELECT s.x FROM (SELECT row_number() OVER (ORDER BY name) AS rn, author AS x FROM tech_book) s",
+ } {
+ resp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
+ Name: database.Name,
+ Statement: statement,
+ }))
+ a.NoError(err, statement)
+ a.Len(resp.Msg.Results, 1, statement)
+ a.Empty(resp.Msg.Results[0].Error, statement)
+ a.NotEmpty(resp.Msg.Results[0].Rows, statement)
+ for _, row := range resp.Msg.Results[0].Rows {
+ a.NotContains([]string{"bber", "Egor Rogov", "Martin Kleppmann"}, row.Values[0].GetStringValue(), statement)
+ }
+ }
+
+ batchResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
+ Name: database.Name,
+ Statement: "SELECT name FROM tech_book; SELECT name FROM tech_book WHERE author = 'bber';",
+ }))
+ a.NoError(err)
+ a.Len(batchResp.Msg.Results, 2)
+ a.Empty(batchResp.Msg.Results[0].Error)
+ a.Len(batchResp.Msg.Results[0].Rows, 3)
+ a.Contains(batchResp.Msg.Results[1].Error, refusedPredicate)
+
// Created directly on the target, after the last schema sync, so the synced
// schema does not know the function or how many columns it returns.
targetDB, err := sql.Open("pgx", fmt.Sprintf("postgresql://postgres:root-password@%s:%s/%s?sslmode=disable", pgContainer.GetHost(), pgContainer.GetPort(), databaseName))
@@ -332,4 +394,40 @@ func TestSensitiveData(t *testing.T) {
a.Len(unmaskedResp.Msg.Results, 1)
a.Equal([]string{"a", "b"}, unmaskedResp.Msg.Results[0].ColumnNames)
a.Len(unmaskedResp.Msg.Results[0].Rows, 1)
+
+ // ghost synced with no columns. A column added outside Bytebase is masked by
+ // name once it syncs, so the condition on it must be refused before the
+ // query runs: a run would take the 30 seconds pg_sleep asks for. The query
+ // syncs the database, so it comes after the unsynced_rows checks.
+ _, err = targetDB.ExecContext(ctx, "ALTER TABLE ghost ADD COLUMN ghost_secret text; INSERT INTO ghost VALUES ('x')")
+ a.NoError(err)
+ workspace, err := ctl.workspaceServiceClient.GetWorkspace(ctx, connect.NewRequest(&v1pb.GetWorkspaceRequest{Name: "workspaces/-"}))
+ a.NoError(err)
+ _, err = ctl.orgPolicyServiceClient.UpdatePolicy(ctx, connect.NewRequest(&v1pb.UpdatePolicyRequest{
+ Policy: &v1pb.Policy{
+ Name: workspace.Msg.Name + "/policies/masking_rule",
+ Type: v1pb.PolicyType_MASKING_RULE,
+ Policy: &v1pb.Policy_MaskingRulePolicy{
+ MaskingRulePolicy: &v1pb.MaskingRulePolicy{
+ Rules: []*v1pb.MaskingRulePolicy_MaskingRule{{
+ Id: "6b1e7c2a-3f4d-4e5a-9b8c-0d1e2f3a4b5c",
+ Condition: &expr.Expr{Expression: `resource.column_name == "ghost_secret"`},
+ SemanticType: "default",
+ }},
+ },
+ },
+ },
+ AllowMissing: true,
+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"masking_rule_policy"}},
+ }))
+ a.NoError(err)
+ start := time.Now()
+ ghostResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{
+ Name: database.Name,
+ Statement: "SELECT 1 FROM ghost WHERE ghost_secret = 'x' AND pg_sleep(30) IS NOT NULL",
+ }))
+ a.NoError(err)
+ a.Less(time.Since(start), 20*time.Second)
+ a.Len(ghostResp.Msg.Results, 1)
+ a.Contains(ghostResp.Msg.Results[0].Error, refusedPredicate)
}
diff --git a/go.mod b/go.mod
index f5eca782da..9825ef498c 100644
--- a/go.mod
+++ b/go.mod
@@ -36,7 +36,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
- github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e
+ github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861
github.com/caarlos0/env/v11 v11.4.1
github.com/cenkalti/backoff/v7 v7.0.0
github.com/cockroachdb/cockroach-go/v2 v2.4.3
diff --git a/go.sum b/go.sum
index 9158695c9f..01e9438af9 100644
--- a/go.sum
+++ b/go.sum
@@ -252,8 +252,8 @@ github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735 h1:v8y6mXeHmYVsdM
github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735/go.mod h1:vTHjcvfn8Pmf65WjnV1ytTCYyC7xNhFjrniv4YWvtA8=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
-github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e h1:HliM6J1o0GjVkQ7j6MarLVhqy0ezGqVrZohVelbxDHU=
-github.com/bytebase/omni v0.0.0-20261020074414-8031142a6c5e/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=
+github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861 h1:IV5+STWCkVNLz6E82uqSeFIZg56uUiVaGpg4ET2kYLk=
+github.com/bytebase/omni v0.0.0-20261023061317-0234ea55a861/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=