Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(masking): refuse a query whose table has no synced columns (#21402)
Vincent Huang487e4285363f
diff --git a/backend/api/v1/query_result_masker.go b/backend/api/v1/query_result_masker.goindex a9718c21ea..b89463579d 100644--- a/backend/api/v1/query_result_masker.go+++ b/backend/api/v1/query_result_masker.go@@ -61,6 +61,13 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbaseif results[i].Error == "" && spans[i].NotFoundError != nil {return errors.Errorf("masking error: %v", spans[i].NotFoundError)}+ // Reject before error handling or masking: results may contain both partial+ // rows and an error, and unresolved lineage cannot safely mask those rows.+ if maskingBlockedByUnresolvedColumns(spans[i], instance) {+ return errors.Errorf(+ "masking cannot be applied: %v, so the query was not returned",+ spans[i].UnresolvedColumnsError)+ }// Skip masking for error result, but redact the error message if the// statement touches masked columns — database errors can contain// actual column values (e.g. "invalid input syntax for type integer: '<value>'").@@ -70,6 +77,8 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbaseif results[i].Error != "" && len(results[i].Rows) == 0 {if i < len(spans) && spans[i] != nil && s.spanTouchesMaskedColumns(ctx, m, instance, user, spans[i]) {results[i].Error = "Query execution failed. Error details are hidden because the query references columns with data masking policies."+ // Structured error fields can contain sensitive values and SQL too.+ results[i].DetailedError = nil}continue}@@ -83,6 +92,15 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbasereturn nil}+// maskingBlockedByUnresolvedColumns keeps the re-sync trigger and masking+// refusal on the same condition, so stale metadata gets a chance to recover.+func maskingBlockedByUnresolvedColumns(span *parserbase.QuerySpan, instance *store.InstanceMessage) bool {+ if span == nil || span.UnresolvedColumnsError == nil || instance == nil {+ return false+ }+ return common.EngineSupportMasking(instance.Metadata.GetEngine())+}+// spanTouchesMaskedColumns checks whether any column referenced anywhere in// the query (SELECT, WHERE, JOIN, etc.) has a masking policy applied.//diff --git a/backend/api/v1/query_result_masker_test.go b/backend/api/v1/query_result_masker_test.gonew file mode 100644index 0000000000..cb54bb3d2a--- /dev/null+++ b/backend/api/v1/query_result_masker_test.go@@ -0,0 +1,89 @@+package v1++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store"+)++func instanceOn(engine storepb.Engine) *store.InstanceMessage {+ return &store.InstanceMessage{+ ResourceID: "inst",+ Metadata: &storepb.Instance{Engine: engine},+ }+}++func unresolvedSpan() *parserbase.QuerySpan {+ return &parserbase.QuerySpan{+ Type: parserbase.Select,+ UnresolvedColumnsError: &parserbase.UnresolvedColumnsError{+ Relations: []parserbase.ColumnResource{+ {Database: "db", Schema: "public", Table: "t"},+ },+ },+ }+}++func TestMaskingBlockedByUnresolvedColumns(t *testing.T) {+ testCases := []struct {+ name string+ span *parserbase.QuerySpan+ instance *store.InstanceMessage+ want bool+ }{+ {+ name: "unresolved columns on a masking engine blocks",+ span: unresolvedSpan(),+ instance: instanceOn(storepb.Engine_POSTGRES),+ want: true,+ },+ {+ // CockroachDB shares the PostgreSQL extractor but does not support masking.+ name: "engine that never masks does not block",+ span: unresolvedSpan(),+ instance: instanceOn(storepb.Engine_COCKROACHDB),+ want: false,+ },+ {+ name: "resolved span does not block",+ span: &parserbase.QuerySpan{Type: parserbase.Select},+ instance: instanceOn(storepb.Engine_POSTGRES),+ want: false,+ },+ {+ name: "nil span does not block",+ span: nil,+ instance: instanceOn(storepb.Engine_POSTGRES),+ want: false,+ },+ {+ name: "nil instance does not block",+ span: unresolvedSpan(),+ instance: nil,+ want: false,+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, maskingBlockedByUnresolvedColumns(tc.span, tc.instance))+ })+ }+}++// These engines must enforce the signal once their extractors produce it.+func TestMaskingEnginesAgreeWithSignalProducers(t *testing.T) {+ for _, engine := range []storepb.Engine{+ storepb.Engine_MYSQL,+ storepb.Engine_ORACLE,+ storepb.Engine_MSSQL,+ storepb.Engine_REDSHIFT,+ } {+ require.True(t, maskingBlockedByUnresolvedColumns(unresolvedSpan(), instanceOn(engine)),+ "%s masks, so it must act on the signal once its extractor sets one", engine)+ }+}diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex c6701cadfe..3c9c76e312 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -741,10 +741,24 @@ func queryRetry(syncDatabaseMap := make(map[string]bool)for i, r := range results {+ if i >= len(spans) {+ continue+ }+ // Re-sync even for error results: MaskResults also refuses partial rows.+ // Permanently empty tables re-sync on every query until BYT-10074 adds throttling.+ if maskingEnabled && maskingBlockedByUnresolvedColumns(spans[i], instance) {+ for _, dbName := range spans[i].UnresolvedColumnsError.Databases() {+ slog.Debug("database metadata need to sync: unresolved columns",+ slog.String("instance", instance.ResourceID),+ slog.String("database", dbName),+ slog.String("detail", spans[i].UnresolvedColumnsError.Error()))+ syncDatabaseMap[dbName] = true+ }+ }if r.Error != "" {continue}- if i < len(spans) && spans[i].NotFoundError != nil {+ if spans[i].NotFoundError != nil {for k := range spans[i].SourceColumns {slog.Debug("database metadata need to sync", slog.String("instance", instance.ResourceID), slog.String("database", k.Database), slog.String("schema", k.Schema), slog.String("table", k.Table), slog.String("column", k.Column))syncDatabaseMap[k.Database] = true@@ -760,11 +774,7 @@ func queryRetry(return nil, nil, duration, err}if d == nil {- // The referenced database is not tracked by Bytebase (e.g. it was- // dropped, excluded by sync filters, or never discovered yet).- // Skip the sync attempt and leave the span's NotFoundError in place- // so the masking policy below can reject the query cleanly instead- // of panicking on a nil *DatabaseMessage.+ // Leave the span error in place so masking can reject an untracked database.slog.Debug("skip metadata sync: database not tracked",slog.String("instance", instance.ResourceID),slog.String("database", accessDatabaseName))diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex bc223c8dfb..a079ee74ba 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -2,6 +2,7 @@ package baseimport ("context"+ "fmt""slices""strings"@@ -73,6 +74,39 @@ type QuerySpan struct {ElasticsearchAnalysis *ElasticsearchAnalysisNotFoundError errorFunctionNotSupportedError error+ // UnresolvedColumnsError prevents masking consumers from treating missing+ // column metadata as an absence of masking policies. Other consumers may ignore it.+ UnresolvedColumnsError *UnresolvedColumnsError+}++// UnresolvedColumnsError identifies relations whose synced metadata has no columns.+type UnresolvedColumnsError struct {+ // Relations each carry an empty Column field.+ Relations []ColumnResource+}++func (e *UnresolvedColumnsError) Error() string {+ names := make([]string, 0, len(e.Relations))+ for _, r := range e.Relations {+ names = append(names, r.String())+ }+ slices.Sort(names)+ return fmt.Sprintf("the synced schema describes no columns for %s", strings.Join(names, ", "))+}++// Databases returns the sorted, distinct database names that need re-syncing.+func (e *UnresolvedColumnsError) Databases() []string {+ seen := make(map[string]bool, len(e.Relations))+ var out []string+ for _, r := range e.Relations {+ if r.Database == "" || seen[r.Database] {+ continue+ }+ seen[r.Database] = true+ out = append(out, r.Database)+ }+ slices.Sort(out)+ return out}// QuerySpanResult is the result column of a query span.diff --git a/backend/plugin/parser/pg/access_tables.go b/backend/plugin/parser/pg/access_tables.goindex a4fe01f578..67c7c1dde3 100644--- a/backend/plugin/parser/pg/access_tables.go+++ b/backend/plugin/parser/pg/access_tables.go@@ -129,7 +129,7 @@ func (a *accessTableExtractor) processRangeVar(rv *ast.RangeVar) {if databaseMetadata == nil {return}- schemaName, name := databaseMetadata.SearchObject(searchPath, resource.Table)+ schemaName, name := databaseMetadata.SearchRelation(searchPath, resource.Table)if schemaName == "" && name == "" {return}diff --git a/backend/plugin/parser/pg/backup.go b/backend/plugin/parser/pg/backup.goindex 43278d5e51..b567b27443 100644--- a/backend/plugin/parser/pg/backup.go+++ b/backend/plugin/parser/pg/backup.go@@ -385,7 +385,7 @@ func extractTableReferenceFromRangeVar(rv *ast.RangeVar, metadata *model.Databasif len(searchPath) == 0 {searchPath = []string{"public"}}- schemaName, _ := metadata.SearchObject(searchPath, rv.Relname)+ schemaName, _ := metadata.SearchRelation(searchPath, rv.Relname)if schemaName == "" {return nil, errors.Errorf("Table %q not found in metadata with search path %v", rv.Relname, searchPath)}diff --git a/backend/plugin/parser/pg/query_span_omni.go b/backend/plugin/parser/pg/query_span_omni.goindex 1cf956a7db..a0208f0656 100644--- a/backend/plugin/parser/pg/query_span_omni.go+++ b/backend/plugin/parser/pg/query_span_omni.go@@ -182,6 +182,50 @@ func findDollarQuotedBody(definition string) (tag string, bodyStart int, bodyEndreturn "", 0, 0}+// unresolvedColumnsError checks stored metadata independently of column lineage.+// Accesses are conservative: CTE names can match physical tables, so this may+// refuse a CTE-only query. Reads missing from the access set remain unchecked.+func (e *omniQuerySpanExtractor) unresolvedColumnsError(accesses base.SourceColumnSet) *base.UnresolvedColumnsError {+ seen := make(map[base.ColumnResource]bool, len(accesses))+ var unresolved []base.ColumnResource+ for access := range accesses {+ relation := base.ColumnResource{+ Server: access.Server,+ Database: access.Database,+ Schema: access.Schema,+ Table: access.Table,+ }+ if relation.Table == "" || seen[relation] {+ continue+ }+ seen[relation] = true+ if e.relationHasNoSyncedColumns(relation) {+ unresolved = append(unresolved, relation)+ }+ }+ if len(unresolved) == 0 {+ return nil+ }+ return &base.UnresolvedColumnsError{Relations: unresolved}+}++// relationHasNoSyncedColumns checks tables only: masking policies attach to+// table columns, and GetTable resolves partitions to their parent.+// A legal zero-column table is indistinguishable from degraded metadata and+// remains blocked even after re-syncing.+func (e *omniQuerySpanExtractor) relationHasNoSyncedColumns(relation base.ColumnResource) bool {+ meta, err := e.getDatabaseMetadata(relation.Database)+ if err != nil || meta == nil {+ return false+ }+ schema := meta.GetSchemaMetadata(relation.Schema)+ if schema == nil {+ return false+ }+ table := schema.GetTable(relation.Table)+ return table != nil && len(table.GetProto().GetColumns()) == 0+}+// getQuerySpan extracts the query span for the given SQL statement.func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string) (*base.QuerySpan, error) {e.ctx = ctx@@ -276,18 +320,20 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)// used as table sources: SELECT * FROM func()).if results := e.tryUserFuncTableSource(selStmt, accessesMap); results != nil {return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessesMap,- Results: results,+ Type: base.Select,+ SourceColumns: accessesMap,+ Results: results,+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),}, nil}// Fail-open: return access tables with best-effort column names and lineage// when analysis fails (e.g., unsupported built-in functions).return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessesMap,- Results: e.extractFallbackColumns(selStmt),- PredicateColumns: e.funcPredicateColumns,+ Type: base.Select,+ SourceColumns: accessesMap,+ Results: e.extractFallbackColumns(selStmt),+ PredicateColumns: e.funcPredicateColumns,+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),}, nil}@@ -302,10 +348,11 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)}return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessesMap,- PredicateColumns: e.funcPredicateColumns,- Results: results,+ Type: base.Select,+ SourceColumns: accessesMap,+ PredicateColumns: e.funcPredicateColumns,+ Results: results,+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),}, nil}diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.gonew file mode 100644index 0000000000..87b8ed7f07--- /dev/null+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go@@ -0,0 +1,262 @@+package pg++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func column(name, typ string) *storepb.ColumnMetadata {+ return &storepb.ColumnMetadata{Name: name, Type: typ}+}++func healthySchema() *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {Name: "t", Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("email", "text"), column("ssn", "text")}},+ {Name: "o", Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("amt", "numeric")}},+ },+ Views: []*storepb.ViewMetadata{{+ Name: "v",+ Definition: "SELECT id, email FROM public.t",+ Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("email", "text")},+ }},+ MaterializedViews: []*storepb.MaterializedViewMetadata{{+ Name: "mv",+ Definition: "SELECT id, ssn FROM public.t",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "t", Column: "id"},+ {Schema: "public", Table: "t", Column: "ssn"},+ },+ }},+ }},+ }+}++// A table with no columns models a snapshot from a sync before #20581.+func degradedSchema() *storepb.DatabaseSchemaMetadata {+ return degradedSchemaNamed("t")+}++func degradedSchemaNamed(table string) *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{Name: table}},+ }},+ }+}++func degradedViewSchema() *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{Name: "t"}},+ Views: []*storepb.ViewMetadata{{Name: "v", Definition: "SELECT id, email FROM public.t"}},+ }},+ }+}++func partialSchema() *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{Name: "t", Columns: []*storepb.ColumnMetadata{column("email", "text"), column("ssn", "text")}}},+ }},+ }+}++func spanFor(t *testing.T, statement string, metadata *storepb.DatabaseSchemaMetadata) *base.QuerySpan {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{+ InstanceID: "inst",+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: statement}, metadata.Name, "", false)+ require.NoError(t, err)+ return span+}++func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {+ statements := []string{+ "SELECT * FROM public.t",+ "SELECT email FROM public.t",+ "SELECT t.email FROM public.t",+ "SELECT * FROM public.t WHERE email = 'x'",+ "SELECT id, email FROM public.t ORDER BY ssn",+ "WITH c AS (SELECT * FROM public.t) SELECT * FROM c",+ // A non-recursive CTE cannot reference itself; the inner t is the physical table.+ "WITH t AS (SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",+ "WITH t AS (SELECT string_agg('x', ',' ORDER BY (SELECT count(*) FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",+ `WITH "T" AS (SELECT 1 AS n) SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) FROM "T"`,+ }+ for _, statement := range statements {+ t.Run(statement, func(t *testing.T) {+ span := spanFor(t, statement, degradedSchema())+ require.NotNil(t, span.UnresolvedColumnsError,+ "a table with no synced columns must mark the span as unresolvable")+ require.Nil(t, span.NotFoundError,+ "this signal exists because NotFoundError does not fire here; if it starts to, one rule is enforced at two points")+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")+ require.Equal(t, []string{"db"}, span.UnresolvedColumnsError.Databases(),+ "the re-sync must target the database holding the unresolved relation")+ })+ }+}++func TestUnresolvedColumnsSignalQuietOnHealthySnapshot(t *testing.T) {+ statements := []string{+ "SELECT * FROM public.t",+ "SELECT email FROM public.t",+ // Signal absence checks snapshot completeness, not masking correctness.+ "SELECT * FROM public.t NATURAL JOIN public.o",+ "SELECT * FROM public.t JOIN public.o USING (id)",+ "SELECT * FROM public.t LEFT JOIN public.o ON t.id = o.id",+ "SELECT *, md5(email) FROM public.t",+ "SELECT * FROM public.v",+ "SELECT * FROM public.mv",+ "WITH c AS (SELECT * FROM public.t) SELECT * FROM c",+ "SELECT * FROM public.t UNION ALL SELECT * FROM public.t",+ "SELECT DISTINCT ON (id) * FROM public.t",+ "SELECT count(*) FROM public.t",+ "SELECT 1",+ "SELECT * FROM generate_series(1, 3)",+ "EXPLAIN SELECT * FROM public.t",+ "EXPLAIN ANALYZE SELECT * FROM public.t",+ "-- plan\nEXPLAIN ANALYZE SELECT * FROM public.t",+ "SHOW search_path",+ "SET search_path TO public",+ }+ for _, statement := range statements {+ t.Run(statement, func(t *testing.T) {+ span := spanFor(t, statement, healthySchema())+ require.Nil(t, span.UnresolvedColumnsError,+ "a fully synced snapshot must never mark a span unresolvable")+ })+ }+}++func TestUnresolvedColumnsSignalScope(t *testing.T) {+ t.Run("partial column loss is out of scope", func(t *testing.T) {+ span := spanFor(t, "SELECT * FROM public.t", partialSchema())+ require.Nil(t, span.UnresolvedColumnsError,+ "a table with some columns resolves; same-arity drift needs the catalog-comparison follow-up")+ })++ t.Run("a view with no synced columns is not refused", func(t *testing.T) {+ span := spanFor(t, "SELECT * FROM public.v", degradedViewSchema())+ require.Nil(t, span.UnresolvedColumnsError)+ })++ t.Run("a foreign table with no synced columns is not refused", func(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ ExternalTables: []*storepb.ExternalTableMetadata{{Name: "ft"}},+ }},+ }+ span := spanFor(t, "SELECT * FROM public.ft", metadata)+ require.Nil(t, span.UnresolvedColumnsError)+ })++ t.Run("a CTE named like a degraded relation is refused too", func(t *testing.T) {+ for _, statement := range []string{+ "WITH t AS (SELECT 1 AS n) SELECT * FROM t",+ "WITH outer_q AS (WITH t AS (SELECT 1 AS n) SELECT * FROM t) SELECT * FROM outer_q",+ "SELECT * FROM (WITH t AS (SELECT 1 AS n) SELECT * FROM t) q",+ "WITH RECURSIVE t AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM t WHERE n < 3) SELECT * FROM t",+ } {+ span := spanFor(t, statement, degradedSchema())+ require.NotNil(t, span.UnresolvedColumnsError, statement)+ }+ })++ t.Run("a non-recursive CTE reading its own name reads the table", func(t *testing.T) {+ span := spanFor(t, "WITH t AS (SELECT * FROM t) SELECT * FROM t", degradedSchema())+ require.NotNil(t, span.UnresolvedColumnsError)+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")+ })++ t.Run("a qualified read is still checked when a CTE shares the name", func(t *testing.T) {+ span := spanFor(t, "WITH t AS (SELECT 1 AS n) SELECT * FROM public.t", degradedSchema())+ require.NotNil(t, span.UnresolvedColumnsError)+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")+ })++ t.Run("a statement reading both the CTE and the qualified table is checked", func(t *testing.T) {+ span := spanFor(t, "WITH t AS (SELECT 1 AS n) SELECT * FROM t UNION ALL SELECT * FROM public.t", degradedSchema())+ require.NotNil(t, span.UnresolvedColumnsError)+ })++ t.Run("materialized view without columns is not a degraded table", func(t *testing.T) {+ span := spanFor(t, "SELECT * FROM public.mv", healthySchema())+ require.Nil(t, span.UnresolvedColumnsError)+ })++ t.Run("error names every unresolved relation", func(t *testing.T) {+ metadata := degradedSchema()+ metadata.Schemas[0].Tables = append(metadata.Schemas[0].Tables, &storepb.TableMetadata{Name: "o"})+ span := spanFor(t, "SELECT * FROM public.t, public.o", metadata)+ require.NotNil(t, span.UnresolvedColumnsError)+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.o")+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")+ })+}++func TestUnresolvedColumnsSignalResolvesRelationsNotRoutines(t *testing.T) {+ // Routines do not shadow relations in PostgreSQL search_path resolution.+ shadowed := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ SearchPath: "a, b",+ Schemas: []*storepb.SchemaMetadata{+ {Name: "a", Functions: []*storepb.FunctionMetadata{{Name: "t", Signature: "t()"}}},+ {Name: "b", Tables: []*storepb.TableMetadata{{Name: "t"}}},+ },+ }+ span := spanFor(t, "SELECT * FROM t", shadowed)+ require.NotNil(t, span.UnresolvedColumnsError,+ "a routine named like the table must not shadow it, or the column-less b.t goes unchecked")+ require.Contains(t, span.UnresolvedColumnsError.Error(), "b.t")++ // Sequences share the relation namespace and do shadow later tables.+ sequenceFirst := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ SearchPath: "a, b",+ Schemas: []*storepb.SchemaMetadata{+ {Name: "a", Sequences: []*storepb.SequenceMetadata{{Name: "t"}}},+ {Name: "b", Tables: []*storepb.TableMetadata{{Name: "t"}}},+ },+ }+ span = spanFor(t, "SELECT * FROM t", sequenceFirst)+ require.Nil(t, span.UnresolvedColumnsError,+ "the sequence in a is the relation this query reads, and a sequence carries no column list to judge")+}++// BYT-10076 tracks reads omitted from the access set; these cases record that gap.+func TestUnresolvedColumnsSignalNotCoveredShapes(t *testing.T) {+ notCovered := map[string]string{+ "subquery in a FROM-clause function argument": "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.d))",+ "subquery inside a VALUES list": "SELECT * FROM (VALUES ((SELECT count(*) FROM public.d))) v(x)",+ }+ for name, statement := range notCovered {+ t.Run(name, func(t *testing.T) {+ span := spanFor(t, statement, degradedSchemaNamed("d"))+ require.Empty(t, span.SourceColumns,+ "the premise of this gap is that the access set is empty; if this fails the gap may have been closed upstream")+ require.Nil(t, span.UnresolvedColumnsError,+ "documented gap: no access reported, so nothing to check (BYT-10076)")+ })+ }+}diff --git a/backend/plugin/parser/redshift/query_span_extractor_omni.go b/backend/plugin/parser/redshift/query_span_extractor_omni.goindex e14f210ca8..a0148eea40 100644--- a/backend/plugin/parser/redshift/query_span_extractor_omni.go+++ b/backend/plugin/parser/redshift/query_span_extractor_omni.go@@ -280,7 +280,7 @@ func (r *redshiftQuerySpanRelationResolver) ResolveRelation(schemaName, relationresolvedSchemaName, resolvedRelationName := schemaName, relationNameif resolvedSchemaName == "" {- resolvedSchemaName, resolvedRelationName = r.metadata.SearchObject(searchPath, relationName)+ resolvedSchemaName, resolvedRelationName = r.metadata.SearchRelation(searchPath, relationName)if resolvedSchemaName == "" && resolvedRelationName == "" {return nil, nil}@@ -439,7 +439,7 @@ func collectRedshiftSchemaQualificationEdits(metadata *model.DatabaseMetadata, sif isOmniCTEReference(rangeVar, ctes) || rangeVar.Loc.Start < 0 {return true}- schemaName, _ := metadata.SearchObject(searchPath, rangeVar.Relname)+ schemaName, _ := metadata.SearchRelation(searchPath, rangeVar.Relname)if schemaName == "" {return true}@@ -1274,18 +1274,18 @@ func (q *omniQuerySpanExtractor) omniAccessTableExists(ctx context.Context, resoreturn false}if resource.Schema == "" {- schemaName, objectName := metadata.SearchObject(q.searchPath, resource.Table)+ schemaName, objectName := metadata.SearchRelation(q.searchPath, resource.Table)return schemaName != "" || objectName != ""}schema := metadata.GetSchemaMetadata(resource.Schema)if schema == nil {return false}+ // Match the relation kinds used by SearchRelation above.return schema.GetTable(resource.Table) != nil ||schema.GetView(resource.Table) != nil ||schema.GetMaterializedView(resource.Table) != nil ||schema.GetExternalTable(resource.Table) != nil ||- schema.GetFunction(resource.Table) != nil ||schema.GetSequence(resource.Table) != nil}@@ -1309,7 +1309,7 @@ func (q *omniQuerySpanExtractor) resolveOmniRangeVar(ctx context.Context, rangeVif metadata == nil {return resource, nil}- schemaName, objectName := metadata.SearchObject(q.searchPath, resource.Table)+ schemaName, objectName := metadata.SearchRelation(q.searchPath, resource.Table)if schemaName == "" && objectName == "" {return base.ColumnResource{}, &base.ResourceNotFoundError{Database: &resource.Database,diff --git a/backend/store/model/pg_searcher.go b/backend/store/model/pg_searcher.goindex d64c833772..2cda6271bd 100644--- a/backend/store/model/pg_searcher.go+++ b/backend/store/model/pg_searcher.go@@ -156,21 +156,6 @@ func (s *DatabaseSearcher) SearchFunctions(name string) ([]string, []*storepb.Fureturn schemas, funcs}-// SearchObject searches for any database object in the search path.-// NOTE: This is primarily designed for PostgreSQL's search_path concept.-func (s *DatabaseSearcher) SearchObject(name string) (string, string) {- for _, schemaName := range s.searchPath {- schema := s.db.GetSchemaMetadata(schemaName)- if schema == nil {- continue- }- if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetFunction(name) != nil || schema.GetProcedure(name) != nil || schema.GetPackage(name) != nil || schema.GetSequence(name) != nil || schema.GetExternalTable(name) != nil {- return schema.proto.Name, name- }- }- return "", ""-}-// SearchTable searches for a table in the search path.// NOTE: This is primarily designed for PostgreSQL's search_path concept.func (d *DatabaseMetadata) SearchTable(searchPath []string, name string) (string, *TableMetadata) {@@ -294,15 +279,16 @@ func (d *DatabaseMetadata) SearchFunctions(searchPath []string, name string) ([]return schemas, funcs}-// SearchObject searches for any database object in the search path.-// NOTE: This is primarily designed for PostgreSQL's search_path concept.-func (d *DatabaseMetadata) SearchObject(searchPath []string, name string) (string, string) {+// SearchRelation resolves a name to the first relation in the search path.+// Routines do not shadow relations; sequences do, because PostgreSQL keeps+// them in the relation namespace.+func (d *DatabaseMetadata) SearchRelation(searchPath []string, name string) (string, string) {for _, schemaName := range searchPath {schema := d.GetSchemaMetadata(schemaName)if schema == nil {continue}- if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetFunction(name) != nil || schema.GetProcedure(name) != nil || schema.GetPackage(name) != nil || schema.GetSequence(name) != nil || schema.GetExternalTable(name) != nil {+ if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetExternalTable(name) != nil || schema.GetSequence(name) != nil {return schema.proto.Name, name}}