Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
revert: "fix(mssql): resolve encrypted/unparseable view columns from synced metadata (BYT-9720)" (#20625) (#20630)
h3n4l9973ebc7e146
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.goindex eacd4639e2..03fcd275d8 100644--- a/backend/api/v1/predicate_columns_check.go+++ b/backend/api/v1/predicate_columns_check.go@@ -81,14 +81,6 @@ func (s *QueryResultMasker) getSensitiveColumnsForPredicate(var result []base.ColumnResourcefor column := range predicateColumns {- if column.UnknownLineage {- // Lineage is unknown (e.g. an encrypted view whose body is hidden), so- // we cannot rule out that the predicate filters on sensitive data.- // Treat it as sensitive to prevent inference leaks such as- // `SELECT COUNT(*) FROM enc_vw WHERE secret = '...'`.- result = append(result, column)- continue- }database := data.getDatabase(column.Database)if database == nil {continuediff --git a/backend/api/v1/query_result_masker.go b/backend/api/v1/query_result_masker.goindex cf889e9b9c..255f2b2fb0 100644--- a/backend/api/v1/query_result_masker.go+++ b/backend/api/v1/query_result_masker.go@@ -434,14 +434,6 @@ func (s *QueryResultMasker) getMaskerForColumnResource(return masker.NewNoneMasker(), nil, nil}- if sourceColumn.UnknownLineage {- // An encrypted/unparseable view column cannot be traced to a base table,- // so its real lineage — and any base-table masking policy — is unknown.- // Fail safe and fully mask. The taint rides on the source column, so this- // one check covers direct refs, expressions, predicates, and subqueries.- return masker.NewDefaultFullMasker(), &MaskingEvaluation{Algorithm: "Full mask", Context: "Unknown column lineage"}, nil- }-database := data.getDatabase(sourceColumn.Database)if database == nil {return masker.NewNoneMasker(), nil, nildiff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex 46eace6c06..bc223c8dfb 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -102,12 +102,6 @@ type ColumnResource struct {Table string// Column is the normalized column name, it should not be empty.Column string- // UnknownLineage marks a source whose lineage could not be traced to a base- // table (e.g. an encrypted/unparseable view whose body is hidden). The masker- // fully masks any result fed by such a source. Because source columns are- // unioned as expressions and subqueries are merged, the taint propagates- // without each merge site having to know about it.- UnknownLineage bool}// String returns the string format of the column resource.@@ -240,10 +234,6 @@ type PhysicalTable struct {Name string// Columns are the columns of the table.Columns []string- // UnknownLineage marks a temp table populated (via SELECT ... INTO) from at- // least one unknown-lineage source. Columns resolved from it inherit the taint- // so the masker still fully masks data laundered through the temp table.- UnknownLineage bool}func (p *PhysicalTable) GetTableName() string {@@ -267,12 +257,11 @@ func (p *PhysicalTable) GetQuerySpanResult() []QuerySpanResult {for _, column := range p.Columns {sourceColumnSet := make(SourceColumnSet, 1)sourceColumnSet[ColumnResource{- Server: p.Server,- Database: p.Database,- Schema: p.Schema,- Table: p.Name,- Column: column,- UnknownLineage: p.UnknownLineage,+ Server: p.Server,+ Database: p.Database,+ Schema: p.Schema,+ Table: p.Name,+ Column: column,}] = trueresult = append(result, QuerySpanResult{Name: column,diff --git a/backend/plugin/parser/tsql/query_span_extractor.go b/backend/plugin/parser/tsql/query_span_extractor.goindex acabd275a7..a544007927 100644--- a/backend/plugin/parser/tsql/query_span_extractor.go+++ b/backend/plugin/parser/tsql/query_span_extractor.go@@ -141,40 +141,7 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabasview := schemaSchema.GetView(viewName)viewColumns, err := q.getColumnsFromCreateView(view.Definition, databaseName, schemaName, viewName)if err != nil {- if !errors.Is(err, errNoCreateViewInDefinition) {- return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)- }- // The definition has no parseable CREATE VIEW body — typically a- // SQL Server WITH ENCRYPTION view, whose source the catalog does- // not expose. Fall back to the synced column metadata (populated- // from sys.columns regardless of encryption) so column resolution- // and SQL Editor browsing keep working. Lineage to the base tables- // is unrecoverable, so each source column is marked UnknownLineage- // and the masker fully masks anything fed by it (fail-safe), rather- // than leak data a base-table masking policy would protect.- cols := view.GetColumns()- if len(cols) == 0 {- // No parseable body AND no synced columns: we cannot even- // enumerate the result shape, so a SELECT * would return- // columns we have no way to mask. Fail closed.- return nil, errors.Errorf("cannot resolve columns for view %s.%s.%s: encrypted/unparseable definition with no synced columns", databaseName, schemaName, viewName)- }- viewColumns = make([]base.QuerySpanResult, 0, len(cols))- for _, col := range cols {- viewColumns = append(viewColumns, base.QuerySpanResult{- Name: col.GetName(),- IsPlainField: true,- SourceColumns: base.SourceColumnSet{- base.ColumnResource{- Database: databaseName,- Schema: schemaName,- Table: viewName,- Column: col.GetName(),- UnknownLineage: true,- }: true,- },- })- }+ return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)}return &base.PhysicalView{Database: databaseName,@@ -192,12 +159,6 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabas}}-// errNoCreateViewInDefinition signals that a view's stored definition contains-// no parseable CREATE VIEW statement. SQL Server returns only a placeholder-// comment for WITH ENCRYPTION views, which parses to zero statements; callers-// detect this sentinel and fall back to the view's synced column metadata.-var errNoCreateViewInDefinition = errors.New("no CREATE VIEW statement found in definition")-// getColumnsFromCreateView parses a CREATE VIEW definition with omni, extracts// the body's result columns, and applies the optional column alias list.func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDatabaseName string, viewSchemaName string, viewName string) ([]base.QuerySpanResult, error) {@@ -218,7 +179,7 @@ func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDat}}if createView == nil {- return nil, errNoCreateViewInDefinition+ return nil, errors.Errorf("no CREATE VIEW statement found in definition")}body, ok := createView.Query.(*ast.SelectStmt)if !ok || body == nil {diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni.go b/backend/plugin/parser/tsql/query_span_extractor_omni.goindex 7206664ddf..af6d0d0c6b 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni.go@@ -803,12 +803,11 @@ func (q *omniQuerySpanExtractor) resolveTableVar(tv *ast.TableVarRef, alias striName: c,SourceColumns: base.SourceColumnSet{base.ColumnResource{- Server: temp.Server,- Database: temp.Database,- Schema: temp.Schema,- Table: temp.Name,- Column: c,- UnknownLineage: temp.UnknownLineage,+ Server: temp.Server,+ Database: temp.Database,+ Schema: temp.Schema,+ Table: temp.Name,+ Column: c,}: true,},IsPlainField: true,@@ -871,24 +870,16 @@ func (q *omniQuerySpanExtractor) populateSelectIntoTempTable(sel *ast.SelectStmtreturn}columns := make([]string, 0, len(results))- unknownLineage := falsefor i, result := range results {name := result.Nameif name == "" && sel.TargetList != nil && i < len(sel.TargetList.Items) {name = q.sliceName(sel.TargetList.Items[i])}columns = append(columns, name)- for src := range result.SourceColumns {- if src.UnknownLineage {- unknownLineage = true- break- }- }}q.gCtx.TempTables[sel.IntoTable.Object] = &base.PhysicalTable{- Name: sel.IntoTable.Object,- Columns: columns,- UnknownLineage: unknownLineage,+ Name: sel.IntoTable.Object,+ Columns: columns,}}diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go b/backend/plugin/parser/tsql/query_span_extractor_omni_test.goindex 53b5677dd2..922d86c6a8 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go@@ -14,7 +14,7 @@ import ()// omniTestMetadata is the default mock catalog used by these tests.-// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and views vw, enc_vw, enc_empty_vw.+// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and a view vw.var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{{Name: "db",@@ -28,20 +28,6 @@ var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{},Views: []*storepb.ViewMetadata{{Name: "vw", Definition: "CREATE VIEW [dbo].[vw] AS SELECT a, b FROM t"},- // enc_vw mimics a SQL Server WITH ENCRYPTION view: the body is not- // retrievable, so sync stores a placeholder comment instead of DDL,- // but the column shape is still synced from sys.columns.- {- Name: "enc_vw",- Definition: "/* Definition of view dbo.enc_vw is encrypted. */",- Columns: []*storepb.ColumnMetadata{{Name: "id"}, {Name: "secret"}},- },- // enc_empty_vw is an encrypted view whose columns were never synced- // (e.g. sync lacked sys.columns access): browsing must still not error.- {- Name: "enc_empty_vw",- Definition: "/* Definition of view dbo.enc_empty_vw is encrypted. */",- },},},},@@ -243,64 +229,6 @@ func TestOmniQuerySpan_SupportedShapes(t *testing.T) {}}-// sourcesUnknownLineage reports whether any source column in the set is flagged-// with unknown lineage.-func sourcesUnknownLineage(set base.SourceColumnSet) bool {- for c := range set {- if c.UnknownLineage {- return true- }- }- return false-}--func TestOmniQuerySpan_EncryptedViewUnknownLineage(t *testing.T) {- // Direct references to an encrypted view's columns resolve from synced- // metadata, but their lineage is unrecoverable, so each source column must be- // flagged UnknownLineage for the masker to fully mask it.- q := newOmniTestExtractor(t, "db")- span, err := q.getOmniQuerySpan(context.Background(), "SELECT id, secret FROM enc_vw")- require.NoError(t, err)- require.Len(t, span.Results, 2)- for _, r := range span.Results {- require.Truef(t, sourcesUnknownLineage(r.SourceColumns), "result %q should have an unknown-lineage source", r.Name)- }-- // The taint must survive an expression that merges sources; otherwise- // `secret + 'x'` over an encrypted column would reach the masker untainted- // and be returned unmasked.- q2 := newOmniTestExtractor(t, "db")- span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret + 'x' AS e FROM enc_vw")- require.NoError(t, err)- require.Len(t, span2.Results, 1)- require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "expression over an encrypted column should stay unknown-lineage")-- // Control: a normal view resolves to real base-table lineage, untainted.- q3 := newOmniTestExtractor(t, "db")- span3, err := q3.getOmniQuerySpan(context.Background(), "SELECT a, b FROM vw")- require.NoError(t, err)- require.NotEmpty(t, span3.Results)- for _, r := range span3.Results {- require.Falsef(t, sourcesUnknownLineage(r.SourceColumns), "result %q from normal view should have known lineage", r.Name)- }-- // A predicate-only reference to an encrypted column must also carry the taint,- // so error redaction over PredicateColumns covers `WHERE secret = ...`.- q4 := newOmniTestExtractor(t, "db")- span4, err := q4.getOmniQuerySpan(context.Background(), "SELECT 1 FROM enc_vw WHERE secret = 'x'")- require.NoError(t, err)- require.True(t, sourcesUnknownLineage(span4.PredicateColumns), "predicate over an encrypted column should be unknown-lineage")-}--// An encrypted view with no synced columns can neither be parsed nor have its-// result shape enumerated, so it must fail closed rather than resolve to columns-// that cannot be masked.-func TestOmniQuerySpan_EncryptedViewNoColumnsFailsClosed(t *testing.T) {- q := newOmniTestExtractor(t, "db")- _, err := q.getOmniQuerySpan(context.Background(), "SELECT * FROM enc_empty_vw")- require.Error(t, err)-}-func TestOmniQuerySpan_NotFound(t *testing.T) {q := newOmniTestExtractor(t, "db")span, err := q.getOmniQuerySpan(context.Background(), "SELECT a FROM no_such_table")@@ -476,30 +404,6 @@ func TestOmniQuerySpan_TempTableDefinitions(t *testing.T) {require.Equal(t, "name", span2.Results[1].Name)}-// SELECT ... INTO #tmp from an encrypted view must carry the unknown-lineage-// taint into the temp table, otherwise a later query of #tmp would launder the-// data past the masker unmasked.-func TestOmniQuerySpan_SelectIntoEncryptedViewKeepsTaint(t *testing.T) {- getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)- gCtx := base.GetQuerySpanContext{- GetDatabaseMetadataFunc: getter,- ListDatabaseNamesFunc: lister,- TempTables: make(map[string]*base.PhysicalTable),- }-- q1 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)- _, err := q1.getOmniQuerySpan(context.Background(), "SELECT secret INTO #tmp FROM enc_vw")- require.NoError(t, err)- require.Contains(t, gCtx.TempTables, "#tmp")- require.True(t, gCtx.TempTables["#tmp"].UnknownLineage, "temp table from an encrypted view should be flagged unknown-lineage")-- q2 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)- span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret FROM #tmp")- require.NoError(t, err)- require.Len(t, span2.Results, 1)- require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "column from a tainted temp table should stay unknown-lineage")-}-func TestOmniQuerySpan_TempTableDefinitionsCaseInsensitive(t *testing.T) {getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)gCtx := base.GetQuerySpanContext{