Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(masking): refuse a query whose table has no synced columns (#21402)

Vincent Huang487e4285363f
diff --git a/backend/api/v1/query_result_masker.go b/backend/api/v1/query_result_masker.go
index a9718c21ea..b89463579d 100644
--- a/backend/api/v1/query_result_masker.go
+++ b/backend/api/v1/query_result_masker.go
@@ -61,6 +61,13 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbase
if results[i].Error == "" && spans[i].NotFoundError != nil {
return errors.Errorf("masking error: %v", spans[i].NotFoundError)
}
+ // Reject before error handling or masking: results may contain both partial
+ // rows and an error, and unresolved lineage cannot safely mask those rows.
+ if maskingBlockedByUnresolvedColumns(spans[i], instance) {
+ return errors.Errorf(
+ "masking cannot be applied: %v, so the query was not returned",
+ spans[i].UnresolvedColumnsError)
+ }
// Skip masking for error result, but redact the error message if the
// statement touches masked columns — database errors can contain
// actual column values (e.g. "invalid input syntax for type integer: '<value>'").
@@ -70,6 +77,8 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbase
if results[i].Error != "" && len(results[i].Rows) == 0 {
if i < len(spans) && spans[i] != nil && s.spanTouchesMaskedColumns(ctx, m, instance, user, spans[i]) {
results[i].Error = "Query execution failed. Error details are hidden because the query references columns with data masking policies."
+ // Structured error fields can contain sensitive values and SQL too.
+ results[i].DetailedError = nil
}
continue
}
@@ -83,6 +92,15 @@ func (s *QueryResultMasker) MaskResults(ctx context.Context, spans []*parserbase
return nil
}
+// maskingBlockedByUnresolvedColumns keeps the re-sync trigger and masking
+// refusal on the same condition, so stale metadata gets a chance to recover.
+func maskingBlockedByUnresolvedColumns(span *parserbase.QuerySpan, instance *store.InstanceMessage) bool {
+ if span == nil || span.UnresolvedColumnsError == nil || instance == nil {
+ return false
+ }
+ return common.EngineSupportMasking(instance.Metadata.GetEngine())
+}
+
// spanTouchesMaskedColumns checks whether any column referenced anywhere in
// the query (SELECT, WHERE, JOIN, etc.) has a masking policy applied.
//
diff --git a/backend/api/v1/query_result_masker_test.go b/backend/api/v1/query_result_masker_test.go
new file mode 100644
index 0000000000..cb54bb3d2a
--- /dev/null
+++ b/backend/api/v1/query_result_masker_test.go
@@ -0,0 +1,89 @@
+package v1
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+func instanceOn(engine storepb.Engine) *store.InstanceMessage {
+ return &store.InstanceMessage{
+ ResourceID: "inst",
+ Metadata: &storepb.Instance{Engine: engine},
+ }
+}
+
+func unresolvedSpan() *parserbase.QuerySpan {
+ return &parserbase.QuerySpan{
+ Type: parserbase.Select,
+ UnresolvedColumnsError: &parserbase.UnresolvedColumnsError{
+ Relations: []parserbase.ColumnResource{
+ {Database: "db", Schema: "public", Table: "t"},
+ },
+ },
+ }
+}
+
+func TestMaskingBlockedByUnresolvedColumns(t *testing.T) {
+ testCases := []struct {
+ name string
+ span *parserbase.QuerySpan
+ instance *store.InstanceMessage
+ want bool
+ }{
+ {
+ name: "unresolved columns on a masking engine blocks",
+ span: unresolvedSpan(),
+ instance: instanceOn(storepb.Engine_POSTGRES),
+ want: true,
+ },
+ {
+ // CockroachDB shares the PostgreSQL extractor but does not support masking.
+ name: "engine that never masks does not block",
+ span: unresolvedSpan(),
+ instance: instanceOn(storepb.Engine_COCKROACHDB),
+ want: false,
+ },
+ {
+ name: "resolved span does not block",
+ span: &parserbase.QuerySpan{Type: parserbase.Select},
+ instance: instanceOn(storepb.Engine_POSTGRES),
+ want: false,
+ },
+ {
+ name: "nil span does not block",
+ span: nil,
+ instance: instanceOn(storepb.Engine_POSTGRES),
+ want: false,
+ },
+ {
+ name: "nil instance does not block",
+ span: unresolvedSpan(),
+ instance: nil,
+ want: false,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ require.Equal(t, tc.want, maskingBlockedByUnresolvedColumns(tc.span, tc.instance))
+ })
+ }
+}
+
+// These engines must enforce the signal once their extractors produce it.
+func TestMaskingEnginesAgreeWithSignalProducers(t *testing.T) {
+ for _, engine := range []storepb.Engine{
+ storepb.Engine_MYSQL,
+ storepb.Engine_ORACLE,
+ storepb.Engine_MSSQL,
+ storepb.Engine_REDSHIFT,
+ } {
+ require.True(t, maskingBlockedByUnresolvedColumns(unresolvedSpan(), instanceOn(engine)),
+ "%s masks, so it must act on the signal once its extractor sets one", engine)
+ }
+}
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index c6701cadfe..3c9c76e312 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -741,10 +741,24 @@ func queryRetry(
syncDatabaseMap := make(map[string]bool)
for i, r := range results {
+ if i >= len(spans) {
+ continue
+ }
+ // Re-sync even for error results: MaskResults also refuses partial rows.
+ // Permanently empty tables re-sync on every query until BYT-10074 adds throttling.
+ if maskingEnabled && maskingBlockedByUnresolvedColumns(spans[i], instance) {
+ for _, dbName := range spans[i].UnresolvedColumnsError.Databases() {
+ slog.Debug("database metadata need to sync: unresolved columns",
+ slog.String("instance", instance.ResourceID),
+ slog.String("database", dbName),
+ slog.String("detail", spans[i].UnresolvedColumnsError.Error()))
+ syncDatabaseMap[dbName] = true
+ }
+ }
if r.Error != "" {
continue
}
- if i < len(spans) && spans[i].NotFoundError != nil {
+ if spans[i].NotFoundError != nil {
for k := range spans[i].SourceColumns {
slog.Debug("database metadata need to sync", slog.String("instance", instance.ResourceID), slog.String("database", k.Database), slog.String("schema", k.Schema), slog.String("table", k.Table), slog.String("column", k.Column))
syncDatabaseMap[k.Database] = true
@@ -760,11 +774,7 @@ func queryRetry(
return nil, nil, duration, err
}
if d == nil {
- // The referenced database is not tracked by Bytebase (e.g. it was
- // dropped, excluded by sync filters, or never discovered yet).
- // Skip the sync attempt and leave the span's NotFoundError in place
- // so the masking policy below can reject the query cleanly instead
- // of panicking on a nil *DatabaseMessage.
+ // Leave the span error in place so masking can reject an untracked database.
slog.Debug("skip metadata sync: database not tracked",
slog.String("instance", instance.ResourceID),
slog.String("database", accessDatabaseName))
diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.go
index bc223c8dfb..a079ee74ba 100644
--- a/backend/plugin/parser/base/span.go
+++ b/backend/plugin/parser/base/span.go
@@ -2,6 +2,7 @@ package base
import (
"context"
+ "fmt"
"slices"
"strings"
@@ -73,6 +74,39 @@ type QuerySpan struct {
ElasticsearchAnalysis *ElasticsearchAnalysis
NotFoundError error
FunctionNotSupportedError error
+ // UnresolvedColumnsError prevents masking consumers from treating missing
+ // column metadata as an absence of masking policies. Other consumers may ignore it.
+ UnresolvedColumnsError *UnresolvedColumnsError
+}
+
+// UnresolvedColumnsError identifies relations whose synced metadata has no columns.
+type UnresolvedColumnsError struct {
+ // Relations each carry an empty Column field.
+ Relations []ColumnResource
+}
+
+func (e *UnresolvedColumnsError) Error() string {
+ names := make([]string, 0, len(e.Relations))
+ for _, r := range e.Relations {
+ names = append(names, r.String())
+ }
+ slices.Sort(names)
+ return fmt.Sprintf("the synced schema describes no columns for %s", strings.Join(names, ", "))
+}
+
+// Databases returns the sorted, distinct database names that need re-syncing.
+func (e *UnresolvedColumnsError) Databases() []string {
+ seen := make(map[string]bool, len(e.Relations))
+ var out []string
+ for _, r := range e.Relations {
+ if r.Database == "" || seen[r.Database] {
+ continue
+ }
+ seen[r.Database] = true
+ out = append(out, r.Database)
+ }
+ slices.Sort(out)
+ return out
}
// QuerySpanResult is the result column of a query span.
diff --git a/backend/plugin/parser/pg/access_tables.go b/backend/plugin/parser/pg/access_tables.go
index a4fe01f578..67c7c1dde3 100644
--- a/backend/plugin/parser/pg/access_tables.go
+++ b/backend/plugin/parser/pg/access_tables.go
@@ -129,7 +129,7 @@ func (a *accessTableExtractor) processRangeVar(rv *ast.RangeVar) {
if databaseMetadata == nil {
return
}
- schemaName, name := databaseMetadata.SearchObject(searchPath, resource.Table)
+ schemaName, name := databaseMetadata.SearchRelation(searchPath, resource.Table)
if schemaName == "" && name == "" {
return
}
diff --git a/backend/plugin/parser/pg/backup.go b/backend/plugin/parser/pg/backup.go
index 43278d5e51..b567b27443 100644
--- a/backend/plugin/parser/pg/backup.go
+++ b/backend/plugin/parser/pg/backup.go
@@ -385,7 +385,7 @@ func extractTableReferenceFromRangeVar(rv *ast.RangeVar, metadata *model.Databas
if len(searchPath) == 0 {
searchPath = []string{"public"}
}
- schemaName, _ := metadata.SearchObject(searchPath, rv.Relname)
+ schemaName, _ := metadata.SearchRelation(searchPath, rv.Relname)
if schemaName == "" {
return nil, errors.Errorf("Table %q not found in metadata with search path %v", rv.Relname, searchPath)
}
diff --git a/backend/plugin/parser/pg/query_span_omni.go b/backend/plugin/parser/pg/query_span_omni.go
index 1cf956a7db..a0208f0656 100644
--- a/backend/plugin/parser/pg/query_span_omni.go
+++ b/backend/plugin/parser/pg/query_span_omni.go
@@ -182,6 +182,50 @@ func findDollarQuotedBody(definition string) (tag string, bodyStart int, bodyEnd
return "", 0, 0
}
+// unresolvedColumnsError checks stored metadata independently of column lineage.
+// Accesses are conservative: CTE names can match physical tables, so this may
+// refuse a CTE-only query. Reads missing from the access set remain unchecked.
+func (e *omniQuerySpanExtractor) unresolvedColumnsError(accesses base.SourceColumnSet) *base.UnresolvedColumnsError {
+ seen := make(map[base.ColumnResource]bool, len(accesses))
+ var unresolved []base.ColumnResource
+ for access := range accesses {
+ relation := base.ColumnResource{
+ Server: access.Server,
+ Database: access.Database,
+ Schema: access.Schema,
+ Table: access.Table,
+ }
+ if relation.Table == "" || seen[relation] {
+ continue
+ }
+ seen[relation] = true
+ if e.relationHasNoSyncedColumns(relation) {
+ unresolved = append(unresolved, relation)
+ }
+ }
+ if len(unresolved) == 0 {
+ return nil
+ }
+ return &base.UnresolvedColumnsError{Relations: unresolved}
+}
+
+// relationHasNoSyncedColumns checks tables only: masking policies attach to
+// table columns, and GetTable resolves partitions to their parent.
+// A legal zero-column table is indistinguishable from degraded metadata and
+// remains blocked even after re-syncing.
+func (e *omniQuerySpanExtractor) relationHasNoSyncedColumns(relation base.ColumnResource) bool {
+ meta, err := e.getDatabaseMetadata(relation.Database)
+ if err != nil || meta == nil {
+ return false
+ }
+ schema := meta.GetSchemaMetadata(relation.Schema)
+ if schema == nil {
+ return false
+ }
+ table := schema.GetTable(relation.Table)
+ return table != nil && len(table.GetProto().GetColumns()) == 0
+}
+
// getQuerySpan extracts the query span for the given SQL statement.
func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string) (*base.QuerySpan, error) {
e.ctx = ctx
@@ -276,18 +320,20 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
// used as table sources: SELECT * FROM func()).
if results := e.tryUserFuncTableSource(selStmt, accessesMap); results != nil {
return &base.QuerySpan{
- Type: base.Select,
- SourceColumns: accessesMap,
- Results: results,
+ Type: base.Select,
+ SourceColumns: accessesMap,
+ Results: results,
+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
}, nil
}
// Fail-open: return access tables with best-effort column names and lineage
// when analysis fails (e.g., unsupported built-in functions).
return &base.QuerySpan{
- Type: base.Select,
- SourceColumns: accessesMap,
- Results: e.extractFallbackColumns(selStmt),
- PredicateColumns: e.funcPredicateColumns,
+ Type: base.Select,
+ SourceColumns: accessesMap,
+ Results: e.extractFallbackColumns(selStmt),
+ PredicateColumns: e.funcPredicateColumns,
+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
}, nil
}
@@ -302,10 +348,11 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
}
return &base.QuerySpan{
- Type: base.Select,
- SourceColumns: accessesMap,
- PredicateColumns: e.funcPredicateColumns,
- Results: results,
+ Type: base.Select,
+ SourceColumns: accessesMap,
+ PredicateColumns: e.funcPredicateColumns,
+ Results: results,
+ UnresolvedColumnsError: e.unresolvedColumnsError(accessesMap),
}, nil
}
diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
new file mode 100644
index 0000000000..87b8ed7f07
--- /dev/null
+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
@@ -0,0 +1,262 @@
+package pg
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func column(name, typ string) *storepb.ColumnMetadata {
+ return &storepb.ColumnMetadata{Name: name, Type: typ}
+}
+
+func healthySchema() *storepb.DatabaseSchemaMetadata {
+ return &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*storepb.TableMetadata{
+ {Name: "t", Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("email", "text"), column("ssn", "text")}},
+ {Name: "o", Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("amt", "numeric")}},
+ },
+ Views: []*storepb.ViewMetadata{{
+ Name: "v",
+ Definition: "SELECT id, email FROM public.t",
+ Columns: []*storepb.ColumnMetadata{column("id", "int4"), column("email", "text")},
+ }},
+ MaterializedViews: []*storepb.MaterializedViewMetadata{{
+ Name: "mv",
+ Definition: "SELECT id, ssn FROM public.t",
+ DependencyColumns: []*storepb.DependencyColumn{
+ {Schema: "public", Table: "t", Column: "id"},
+ {Schema: "public", Table: "t", Column: "ssn"},
+ },
+ }},
+ }},
+ }
+}
+
+// A table with no columns models a snapshot from a sync before #20581.
+func degradedSchema() *storepb.DatabaseSchemaMetadata {
+ return degradedSchemaNamed("t")
+}
+
+func degradedSchemaNamed(table string) *storepb.DatabaseSchemaMetadata {
+ return &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*storepb.TableMetadata{{Name: table}},
+ }},
+ }
+}
+
+func degradedViewSchema() *storepb.DatabaseSchemaMetadata {
+ return &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*storepb.TableMetadata{{Name: "t"}},
+ Views: []*storepb.ViewMetadata{{Name: "v", Definition: "SELECT id, email FROM public.t"}},
+ }},
+ }
+}
+
+func partialSchema() *storepb.DatabaseSchemaMetadata {
+ return &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*storepb.TableMetadata{{Name: "t", Columns: []*storepb.ColumnMetadata{column("email", "text"), column("ssn", "text")}}},
+ }},
+ }
+}
+
+func spanFor(t *testing.T, statement string, metadata *storepb.DatabaseSchemaMetadata) *base.QuerySpan {
+ t.Helper()
+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})
+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{
+ InstanceID: "inst",
+ GetDatabaseMetadataFunc: getter,
+ ListDatabaseNamesFunc: lister,
+ }, base.Statement{Text: statement}, metadata.Name, "", false)
+ require.NoError(t, err)
+ return span
+}
+
+func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {
+ statements := []string{
+ "SELECT * FROM public.t",
+ "SELECT email FROM public.t",
+ "SELECT t.email FROM public.t",
+ "SELECT * FROM public.t WHERE email = 'x'",
+ "SELECT id, email FROM public.t ORDER BY ssn",
+ "WITH c AS (SELECT * FROM public.t) SELECT * FROM c",
+ // A non-recursive CTE cannot reference itself; the inner t is the physical table.
+ "WITH t AS (SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",
+ "WITH t AS (SELECT string_agg('x', ',' ORDER BY (SELECT count(*) FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",
+ `WITH "T" AS (SELECT 1 AS n) SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) FROM "T"`,
+ }
+ for _, statement := range statements {
+ t.Run(statement, func(t *testing.T) {
+ span := spanFor(t, statement, degradedSchema())
+ require.NotNil(t, span.UnresolvedColumnsError,
+ "a table with no synced columns must mark the span as unresolvable")
+ require.Nil(t, span.NotFoundError,
+ "this signal exists because NotFoundError does not fire here; if it starts to, one rule is enforced at two points")
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")
+ require.Equal(t, []string{"db"}, span.UnresolvedColumnsError.Databases(),
+ "the re-sync must target the database holding the unresolved relation")
+ })
+ }
+}
+
+func TestUnresolvedColumnsSignalQuietOnHealthySnapshot(t *testing.T) {
+ statements := []string{
+ "SELECT * FROM public.t",
+ "SELECT email FROM public.t",
+ // Signal absence checks snapshot completeness, not masking correctness.
+ "SELECT * FROM public.t NATURAL JOIN public.o",
+ "SELECT * FROM public.t JOIN public.o USING (id)",
+ "SELECT * FROM public.t LEFT JOIN public.o ON t.id = o.id",
+ "SELECT *, md5(email) FROM public.t",
+ "SELECT * FROM public.v",
+ "SELECT * FROM public.mv",
+ "WITH c AS (SELECT * FROM public.t) SELECT * FROM c",
+ "SELECT * FROM public.t UNION ALL SELECT * FROM public.t",
+ "SELECT DISTINCT ON (id) * FROM public.t",
+ "SELECT count(*) FROM public.t",
+ "SELECT 1",
+ "SELECT * FROM generate_series(1, 3)",
+ "EXPLAIN SELECT * FROM public.t",
+ "EXPLAIN ANALYZE SELECT * FROM public.t",
+ "-- plan\nEXPLAIN ANALYZE SELECT * FROM public.t",
+ "SHOW search_path",
+ "SET search_path TO public",
+ }
+ for _, statement := range statements {
+ t.Run(statement, func(t *testing.T) {
+ span := spanFor(t, statement, healthySchema())
+ require.Nil(t, span.UnresolvedColumnsError,
+ "a fully synced snapshot must never mark a span unresolvable")
+ })
+ }
+}
+
+func TestUnresolvedColumnsSignalScope(t *testing.T) {
+ t.Run("partial column loss is out of scope", func(t *testing.T) {
+ span := spanFor(t, "SELECT * FROM public.t", partialSchema())
+ require.Nil(t, span.UnresolvedColumnsError,
+ "a table with some columns resolves; same-arity drift needs the catalog-comparison follow-up")
+ })
+
+ t.Run("a view with no synced columns is not refused", func(t *testing.T) {
+ span := spanFor(t, "SELECT * FROM public.v", degradedViewSchema())
+ require.Nil(t, span.UnresolvedColumnsError)
+ })
+
+ t.Run("a foreign table with no synced columns is not refused", func(t *testing.T) {
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: "public",
+ ExternalTables: []*storepb.ExternalTableMetadata{{Name: "ft"}},
+ }},
+ }
+ span := spanFor(t, "SELECT * FROM public.ft", metadata)
+ require.Nil(t, span.UnresolvedColumnsError)
+ })
+
+ t.Run("a CTE named like a degraded relation is refused too", func(t *testing.T) {
+ for _, statement := range []string{
+ "WITH t AS (SELECT 1 AS n) SELECT * FROM t",
+ "WITH outer_q AS (WITH t AS (SELECT 1 AS n) SELECT * FROM t) SELECT * FROM outer_q",
+ "SELECT * FROM (WITH t AS (SELECT 1 AS n) SELECT * FROM t) q",
+ "WITH RECURSIVE t AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM t WHERE n < 3) SELECT * FROM t",
+ } {
+ span := spanFor(t, statement, degradedSchema())
+ require.NotNil(t, span.UnresolvedColumnsError, statement)
+ }
+ })
+
+ t.Run("a non-recursive CTE reading its own name reads the table", func(t *testing.T) {
+ span := spanFor(t, "WITH t AS (SELECT * FROM t) SELECT * FROM t", degradedSchema())
+ require.NotNil(t, span.UnresolvedColumnsError)
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")
+ })
+
+ t.Run("a qualified read is still checked when a CTE shares the name", func(t *testing.T) {
+ span := spanFor(t, "WITH t AS (SELECT 1 AS n) SELECT * FROM public.t", degradedSchema())
+ require.NotNil(t, span.UnresolvedColumnsError)
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")
+ })
+
+ t.Run("a statement reading both the CTE and the qualified table is checked", func(t *testing.T) {
+ span := spanFor(t, "WITH t AS (SELECT 1 AS n) SELECT * FROM t UNION ALL SELECT * FROM public.t", degradedSchema())
+ require.NotNil(t, span.UnresolvedColumnsError)
+ })
+
+ t.Run("materialized view without columns is not a degraded table", func(t *testing.T) {
+ span := spanFor(t, "SELECT * FROM public.mv", healthySchema())
+ require.Nil(t, span.UnresolvedColumnsError)
+ })
+
+ t.Run("error names every unresolved relation", func(t *testing.T) {
+ metadata := degradedSchema()
+ metadata.Schemas[0].Tables = append(metadata.Schemas[0].Tables, &storepb.TableMetadata{Name: "o"})
+ span := spanFor(t, "SELECT * FROM public.t, public.o", metadata)
+ require.NotNil(t, span.UnresolvedColumnsError)
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.o")
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "public.t")
+ })
+}
+
+func TestUnresolvedColumnsSignalResolvesRelationsNotRoutines(t *testing.T) {
+ // Routines do not shadow relations in PostgreSQL search_path resolution.
+ shadowed := &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ SearchPath: "a, b",
+ Schemas: []*storepb.SchemaMetadata{
+ {Name: "a", Functions: []*storepb.FunctionMetadata{{Name: "t", Signature: "t()"}}},
+ {Name: "b", Tables: []*storepb.TableMetadata{{Name: "t"}}},
+ },
+ }
+ span := spanFor(t, "SELECT * FROM t", shadowed)
+ require.NotNil(t, span.UnresolvedColumnsError,
+ "a routine named like the table must not shadow it, or the column-less b.t goes unchecked")
+ require.Contains(t, span.UnresolvedColumnsError.Error(), "b.t")
+
+ // Sequences share the relation namespace and do shadow later tables.
+ sequenceFirst := &storepb.DatabaseSchemaMetadata{
+ Name: "db",
+ SearchPath: "a, b",
+ Schemas: []*storepb.SchemaMetadata{
+ {Name: "a", Sequences: []*storepb.SequenceMetadata{{Name: "t"}}},
+ {Name: "b", Tables: []*storepb.TableMetadata{{Name: "t"}}},
+ },
+ }
+ span = spanFor(t, "SELECT * FROM t", sequenceFirst)
+ require.Nil(t, span.UnresolvedColumnsError,
+ "the sequence in a is the relation this query reads, and a sequence carries no column list to judge")
+}
+
+// BYT-10076 tracks reads omitted from the access set; these cases record that gap.
+func TestUnresolvedColumnsSignalNotCoveredShapes(t *testing.T) {
+ notCovered := map[string]string{
+ "subquery in a FROM-clause function argument": "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.d))",
+ "subquery inside a VALUES list": "SELECT * FROM (VALUES ((SELECT count(*) FROM public.d))) v(x)",
+ }
+ for name, statement := range notCovered {
+ t.Run(name, func(t *testing.T) {
+ span := spanFor(t, statement, degradedSchemaNamed("d"))
+ require.Empty(t, span.SourceColumns,
+ "the premise of this gap is that the access set is empty; if this fails the gap may have been closed upstream")
+ require.Nil(t, span.UnresolvedColumnsError,
+ "documented gap: no access reported, so nothing to check (BYT-10076)")
+ })
+ }
+}
diff --git a/backend/plugin/parser/redshift/query_span_extractor_omni.go b/backend/plugin/parser/redshift/query_span_extractor_omni.go
index e14f210ca8..a0148eea40 100644
--- a/backend/plugin/parser/redshift/query_span_extractor_omni.go
+++ b/backend/plugin/parser/redshift/query_span_extractor_omni.go
@@ -280,7 +280,7 @@ func (r *redshiftQuerySpanRelationResolver) ResolveRelation(schemaName, relation
resolvedSchemaName, resolvedRelationName := schemaName, relationName
if resolvedSchemaName == "" {
- resolvedSchemaName, resolvedRelationName = r.metadata.SearchObject(searchPath, relationName)
+ resolvedSchemaName, resolvedRelationName = r.metadata.SearchRelation(searchPath, relationName)
if resolvedSchemaName == "" && resolvedRelationName == "" {
return nil, nil
}
@@ -439,7 +439,7 @@ func collectRedshiftSchemaQualificationEdits(metadata *model.DatabaseMetadata, s
if isOmniCTEReference(rangeVar, ctes) || rangeVar.Loc.Start < 0 {
return true
}
- schemaName, _ := metadata.SearchObject(searchPath, rangeVar.Relname)
+ schemaName, _ := metadata.SearchRelation(searchPath, rangeVar.Relname)
if schemaName == "" {
return true
}
@@ -1274,18 +1274,18 @@ func (q *omniQuerySpanExtractor) omniAccessTableExists(ctx context.Context, reso
return false
}
if resource.Schema == "" {
- schemaName, objectName := metadata.SearchObject(q.searchPath, resource.Table)
+ schemaName, objectName := metadata.SearchRelation(q.searchPath, resource.Table)
return schemaName != "" || objectName != ""
}
schema := metadata.GetSchemaMetadata(resource.Schema)
if schema == nil {
return false
}
+ // Match the relation kinds used by SearchRelation above.
return schema.GetTable(resource.Table) != nil ||
schema.GetView(resource.Table) != nil ||
schema.GetMaterializedView(resource.Table) != nil ||
schema.GetExternalTable(resource.Table) != nil ||
- schema.GetFunction(resource.Table) != nil ||
schema.GetSequence(resource.Table) != nil
}
@@ -1309,7 +1309,7 @@ func (q *omniQuerySpanExtractor) resolveOmniRangeVar(ctx context.Context, rangeV
if metadata == nil {
return resource, nil
}
- schemaName, objectName := metadata.SearchObject(q.searchPath, resource.Table)
+ schemaName, objectName := metadata.SearchRelation(q.searchPath, resource.Table)
if schemaName == "" && objectName == "" {
return base.ColumnResource{}, &base.ResourceNotFoundError{
Database: &resource.Database,
diff --git a/backend/store/model/pg_searcher.go b/backend/store/model/pg_searcher.go
index d64c833772..2cda6271bd 100644
--- a/backend/store/model/pg_searcher.go
+++ b/backend/store/model/pg_searcher.go
@@ -156,21 +156,6 @@ func (s *DatabaseSearcher) SearchFunctions(name string) ([]string, []*storepb.Fu
return schemas, funcs
}
-// SearchObject searches for any database object in the search path.
-// NOTE: This is primarily designed for PostgreSQL's search_path concept.
-func (s *DatabaseSearcher) SearchObject(name string) (string, string) {
- for _, schemaName := range s.searchPath {
- schema := s.db.GetSchemaMetadata(schemaName)
- if schema == nil {
- continue
- }
- if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetFunction(name) != nil || schema.GetProcedure(name) != nil || schema.GetPackage(name) != nil || schema.GetSequence(name) != nil || schema.GetExternalTable(name) != nil {
- return schema.proto.Name, name
- }
- }
- return "", ""
-}
-
// SearchTable searches for a table in the search path.
// NOTE: This is primarily designed for PostgreSQL's search_path concept.
func (d *DatabaseMetadata) SearchTable(searchPath []string, name string) (string, *TableMetadata) {
@@ -294,15 +279,16 @@ func (d *DatabaseMetadata) SearchFunctions(searchPath []string, name string) ([]
return schemas, funcs
}
-// SearchObject searches for any database object in the search path.
-// NOTE: This is primarily designed for PostgreSQL's search_path concept.
-func (d *DatabaseMetadata) SearchObject(searchPath []string, name string) (string, string) {
+// SearchRelation resolves a name to the first relation in the search path.
+// Routines do not shadow relations; sequences do, because PostgreSQL keeps
+// them in the relation namespace.
+func (d *DatabaseMetadata) SearchRelation(searchPath []string, name string) (string, string) {
for _, schemaName := range searchPath {
schema := d.GetSchemaMetadata(schemaName)
if schema == nil {
continue
}
- if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetFunction(name) != nil || schema.GetProcedure(name) != nil || schema.GetPackage(name) != nil || schema.GetSequence(name) != nil || schema.GetExternalTable(name) != nil {
+ if schema.GetTable(name) != nil || schema.GetView(name) != nil || schema.GetMaterializedView(name) != nil || schema.GetExternalTable(name) != nil || schema.GetSequence(name) != nil {
return schema.proto.Name, name
}
}