Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

revert: "fix(mssql): resolve encrypted/unparseable view columns from synced metadata (BYT-9720)" (#20625) (#20630)

h3n4l9973ebc7e146
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.go
index eacd4639e2..03fcd275d8 100644
--- a/backend/api/v1/predicate_columns_check.go
+++ b/backend/api/v1/predicate_columns_check.go
@@ -81,14 +81,6 @@ func (s *QueryResultMasker) getSensitiveColumnsForPredicate(
var result []base.ColumnResource
for column := range predicateColumns {
- if column.UnknownLineage {
- // Lineage is unknown (e.g. an encrypted view whose body is hidden), so
- // we cannot rule out that the predicate filters on sensitive data.
- // Treat it as sensitive to prevent inference leaks such as
- // `SELECT COUNT(*) FROM enc_vw WHERE secret = '...'`.
- result = append(result, column)
- continue
- }
database := data.getDatabase(column.Database)
if database == nil {
continue
diff --git a/backend/api/v1/query_result_masker.go b/backend/api/v1/query_result_masker.go
index cf889e9b9c..255f2b2fb0 100644
--- a/backend/api/v1/query_result_masker.go
+++ b/backend/api/v1/query_result_masker.go
@@ -434,14 +434,6 @@ func (s *QueryResultMasker) getMaskerForColumnResource(
return masker.NewNoneMasker(), nil, nil
}
- if sourceColumn.UnknownLineage {
- // An encrypted/unparseable view column cannot be traced to a base table,
- // so its real lineage — and any base-table masking policy — is unknown.
- // Fail safe and fully mask. The taint rides on the source column, so this
- // one check covers direct refs, expressions, predicates, and subqueries.
- return masker.NewDefaultFullMasker(), &MaskingEvaluation{Algorithm: "Full mask", Context: "Unknown column lineage"}, nil
- }
-
database := data.getDatabase(sourceColumn.Database)
if database == nil {
return masker.NewNoneMasker(), nil, nil
diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.go
index 46eace6c06..bc223c8dfb 100644
--- a/backend/plugin/parser/base/span.go
+++ b/backend/plugin/parser/base/span.go
@@ -102,12 +102,6 @@ type ColumnResource struct {
Table string
// Column is the normalized column name, it should not be empty.
Column string
- // UnknownLineage marks a source whose lineage could not be traced to a base
- // table (e.g. an encrypted/unparseable view whose body is hidden). The masker
- // fully masks any result fed by such a source. Because source columns are
- // unioned as expressions and subqueries are merged, the taint propagates
- // without each merge site having to know about it.
- UnknownLineage bool
}
// String returns the string format of the column resource.
@@ -240,10 +234,6 @@ type PhysicalTable struct {
Name string
// Columns are the columns of the table.
Columns []string
- // UnknownLineage marks a temp table populated (via SELECT ... INTO) from at
- // least one unknown-lineage source. Columns resolved from it inherit the taint
- // so the masker still fully masks data laundered through the temp table.
- UnknownLineage bool
}
func (p *PhysicalTable) GetTableName() string {
@@ -267,12 +257,11 @@ func (p *PhysicalTable) GetQuerySpanResult() []QuerySpanResult {
for _, column := range p.Columns {
sourceColumnSet := make(SourceColumnSet, 1)
sourceColumnSet[ColumnResource{
- Server: p.Server,
- Database: p.Database,
- Schema: p.Schema,
- Table: p.Name,
- Column: column,
- UnknownLineage: p.UnknownLineage,
+ Server: p.Server,
+ Database: p.Database,
+ Schema: p.Schema,
+ Table: p.Name,
+ Column: column,
}] = true
result = append(result, QuerySpanResult{
Name: column,
diff --git a/backend/plugin/parser/tsql/query_span_extractor.go b/backend/plugin/parser/tsql/query_span_extractor.go
index acabd275a7..a544007927 100644
--- a/backend/plugin/parser/tsql/query_span_extractor.go
+++ b/backend/plugin/parser/tsql/query_span_extractor.go
@@ -141,40 +141,7 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabas
view := schemaSchema.GetView(viewName)
viewColumns, err := q.getColumnsFromCreateView(view.Definition, databaseName, schemaName, viewName)
if err != nil {
- if !errors.Is(err, errNoCreateViewInDefinition) {
- return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)
- }
- // The definition has no parseable CREATE VIEW body — typically a
- // SQL Server WITH ENCRYPTION view, whose source the catalog does
- // not expose. Fall back to the synced column metadata (populated
- // from sys.columns regardless of encryption) so column resolution
- // and SQL Editor browsing keep working. Lineage to the base tables
- // is unrecoverable, so each source column is marked UnknownLineage
- // and the masker fully masks anything fed by it (fail-safe), rather
- // than leak data a base-table masking policy would protect.
- cols := view.GetColumns()
- if len(cols) == 0 {
- // No parseable body AND no synced columns: we cannot even
- // enumerate the result shape, so a SELECT * would return
- // columns we have no way to mask. Fail closed.
- return nil, errors.Errorf("cannot resolve columns for view %s.%s.%s: encrypted/unparseable definition with no synced columns", databaseName, schemaName, viewName)
- }
- viewColumns = make([]base.QuerySpanResult, 0, len(cols))
- for _, col := range cols {
- viewColumns = append(viewColumns, base.QuerySpanResult{
- Name: col.GetName(),
- IsPlainField: true,
- SourceColumns: base.SourceColumnSet{
- base.ColumnResource{
- Database: databaseName,
- Schema: schemaName,
- Table: viewName,
- Column: col.GetName(),
- UnknownLineage: true,
- }: true,
- },
- })
- }
+ return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)
}
return &base.PhysicalView{
Database: databaseName,
@@ -192,12 +159,6 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabas
}
}
-// errNoCreateViewInDefinition signals that a view's stored definition contains
-// no parseable CREATE VIEW statement. SQL Server returns only a placeholder
-// comment for WITH ENCRYPTION views, which parses to zero statements; callers
-// detect this sentinel and fall back to the view's synced column metadata.
-var errNoCreateViewInDefinition = errors.New("no CREATE VIEW statement found in definition")
-
// getColumnsFromCreateView parses a CREATE VIEW definition with omni, extracts
// the body's result columns, and applies the optional column alias list.
func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDatabaseName string, viewSchemaName string, viewName string) ([]base.QuerySpanResult, error) {
@@ -218,7 +179,7 @@ func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDat
}
}
if createView == nil {
- return nil, errNoCreateViewInDefinition
+ return nil, errors.Errorf("no CREATE VIEW statement found in definition")
}
body, ok := createView.Query.(*ast.SelectStmt)
if !ok || body == nil {
diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni.go b/backend/plugin/parser/tsql/query_span_extractor_omni.go
index 7206664ddf..af6d0d0c6b 100644
--- a/backend/plugin/parser/tsql/query_span_extractor_omni.go
+++ b/backend/plugin/parser/tsql/query_span_extractor_omni.go
@@ -803,12 +803,11 @@ func (q *omniQuerySpanExtractor) resolveTableVar(tv *ast.TableVarRef, alias stri
Name: c,
SourceColumns: base.SourceColumnSet{
base.ColumnResource{
- Server: temp.Server,
- Database: temp.Database,
- Schema: temp.Schema,
- Table: temp.Name,
- Column: c,
- UnknownLineage: temp.UnknownLineage,
+ Server: temp.Server,
+ Database: temp.Database,
+ Schema: temp.Schema,
+ Table: temp.Name,
+ Column: c,
}: true,
},
IsPlainField: true,
@@ -871,24 +870,16 @@ func (q *omniQuerySpanExtractor) populateSelectIntoTempTable(sel *ast.SelectStmt
return
}
columns := make([]string, 0, len(results))
- unknownLineage := false
for i, result := range results {
name := result.Name
if name == "" && sel.TargetList != nil && i < len(sel.TargetList.Items) {
name = q.sliceName(sel.TargetList.Items[i])
}
columns = append(columns, name)
- for src := range result.SourceColumns {
- if src.UnknownLineage {
- unknownLineage = true
- break
- }
- }
}
q.gCtx.TempTables[sel.IntoTable.Object] = &base.PhysicalTable{
- Name: sel.IntoTable.Object,
- Columns: columns,
- UnknownLineage: unknownLineage,
+ Name: sel.IntoTable.Object,
+ Columns: columns,
}
}
diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
index 53b5677dd2..922d86c6a8 100644
--- a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
+++ b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
@@ -14,7 +14,7 @@ import (
)
// omniTestMetadata is the default mock catalog used by these tests.
-// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and views vw, enc_vw, enc_empty_vw.
+// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and a view vw.
var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{
{
Name: "db",
@@ -28,20 +28,6 @@ var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{
},
Views: []*storepb.ViewMetadata{
{Name: "vw", Definition: "CREATE VIEW [dbo].[vw] AS SELECT a, b FROM t"},
- // enc_vw mimics a SQL Server WITH ENCRYPTION view: the body is not
- // retrievable, so sync stores a placeholder comment instead of DDL,
- // but the column shape is still synced from sys.columns.
- {
- Name: "enc_vw",
- Definition: "/* Definition of view dbo.enc_vw is encrypted. */",
- Columns: []*storepb.ColumnMetadata{{Name: "id"}, {Name: "secret"}},
- },
- // enc_empty_vw is an encrypted view whose columns were never synced
- // (e.g. sync lacked sys.columns access): browsing must still not error.
- {
- Name: "enc_empty_vw",
- Definition: "/* Definition of view dbo.enc_empty_vw is encrypted. */",
- },
},
},
},
@@ -243,64 +229,6 @@ func TestOmniQuerySpan_SupportedShapes(t *testing.T) {
}
}
-// sourcesUnknownLineage reports whether any source column in the set is flagged
-// with unknown lineage.
-func sourcesUnknownLineage(set base.SourceColumnSet) bool {
- for c := range set {
- if c.UnknownLineage {
- return true
- }
- }
- return false
-}
-
-func TestOmniQuerySpan_EncryptedViewUnknownLineage(t *testing.T) {
- // Direct references to an encrypted view's columns resolve from synced
- // metadata, but their lineage is unrecoverable, so each source column must be
- // flagged UnknownLineage for the masker to fully mask it.
- q := newOmniTestExtractor(t, "db")
- span, err := q.getOmniQuerySpan(context.Background(), "SELECT id, secret FROM enc_vw")
- require.NoError(t, err)
- require.Len(t, span.Results, 2)
- for _, r := range span.Results {
- require.Truef(t, sourcesUnknownLineage(r.SourceColumns), "result %q should have an unknown-lineage source", r.Name)
- }
-
- // The taint must survive an expression that merges sources; otherwise
- // `secret + 'x'` over an encrypted column would reach the masker untainted
- // and be returned unmasked.
- q2 := newOmniTestExtractor(t, "db")
- span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret + 'x' AS e FROM enc_vw")
- require.NoError(t, err)
- require.Len(t, span2.Results, 1)
- require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "expression over an encrypted column should stay unknown-lineage")
-
- // Control: a normal view resolves to real base-table lineage, untainted.
- q3 := newOmniTestExtractor(t, "db")
- span3, err := q3.getOmniQuerySpan(context.Background(), "SELECT a, b FROM vw")
- require.NoError(t, err)
- require.NotEmpty(t, span3.Results)
- for _, r := range span3.Results {
- require.Falsef(t, sourcesUnknownLineage(r.SourceColumns), "result %q from normal view should have known lineage", r.Name)
- }
-
- // A predicate-only reference to an encrypted column must also carry the taint,
- // so error redaction over PredicateColumns covers `WHERE secret = ...`.
- q4 := newOmniTestExtractor(t, "db")
- span4, err := q4.getOmniQuerySpan(context.Background(), "SELECT 1 FROM enc_vw WHERE secret = 'x'")
- require.NoError(t, err)
- require.True(t, sourcesUnknownLineage(span4.PredicateColumns), "predicate over an encrypted column should be unknown-lineage")
-}
-
-// An encrypted view with no synced columns can neither be parsed nor have its
-// result shape enumerated, so it must fail closed rather than resolve to columns
-// that cannot be masked.
-func TestOmniQuerySpan_EncryptedViewNoColumnsFailsClosed(t *testing.T) {
- q := newOmniTestExtractor(t, "db")
- _, err := q.getOmniQuerySpan(context.Background(), "SELECT * FROM enc_empty_vw")
- require.Error(t, err)
-}
-
func TestOmniQuerySpan_NotFound(t *testing.T) {
q := newOmniTestExtractor(t, "db")
span, err := q.getOmniQuerySpan(context.Background(), "SELECT a FROM no_such_table")
@@ -476,30 +404,6 @@ func TestOmniQuerySpan_TempTableDefinitions(t *testing.T) {
require.Equal(t, "name", span2.Results[1].Name)
}
-// SELECT ... INTO #tmp from an encrypted view must carry the unknown-lineage
-// taint into the temp table, otherwise a later query of #tmp would launder the
-// data past the masker unmasked.
-func TestOmniQuerySpan_SelectIntoEncryptedViewKeepsTaint(t *testing.T) {
- getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)
- gCtx := base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: getter,
- ListDatabaseNamesFunc: lister,
- TempTables: make(map[string]*base.PhysicalTable),
- }
-
- q1 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)
- _, err := q1.getOmniQuerySpan(context.Background(), "SELECT secret INTO #tmp FROM enc_vw")
- require.NoError(t, err)
- require.Contains(t, gCtx.TempTables, "#tmp")
- require.True(t, gCtx.TempTables["#tmp"].UnknownLineage, "temp table from an encrypted view should be flagged unknown-lineage")
-
- q2 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)
- span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret FROM #tmp")
- require.NoError(t, err)
- require.Len(t, span2.Results, 1)
- require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "column from a tainted temp table should stay unknown-lineage")
-}
-
func TestOmniQuerySpan_TempTableDefinitionsCaseInsensitive(t *testing.T) {
getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)
gCtx := base.GetQuerySpanContext{