Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(mssql): resolve encrypted/unparseable view columns from synced metadata (BYT-9720) (#20625)
h3n4l746c44b4eb4d
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.goindex 03fcd275d8..eacd4639e2 100644--- a/backend/api/v1/predicate_columns_check.go+++ b/backend/api/v1/predicate_columns_check.go@@ -81,6 +81,14 @@ func (s *QueryResultMasker) getSensitiveColumnsForPredicate(var result []base.ColumnResourcefor column := range predicateColumns {+ if column.UnknownLineage {+ // Lineage is unknown (e.g. an encrypted view whose body is hidden), so+ // we cannot rule out that the predicate filters on sensitive data.+ // Treat it as sensitive to prevent inference leaks such as+ // `SELECT COUNT(*) FROM enc_vw WHERE secret = '...'`.+ result = append(result, column)+ continue+ }database := data.getDatabase(column.Database)if database == nil {continuediff --git a/backend/api/v1/query_result_masker.go b/backend/api/v1/query_result_masker.goindex 255f2b2fb0..cf889e9b9c 100644--- a/backend/api/v1/query_result_masker.go+++ b/backend/api/v1/query_result_masker.go@@ -434,6 +434,14 @@ func (s *QueryResultMasker) getMaskerForColumnResource(return masker.NewNoneMasker(), nil, nil}+ if sourceColumn.UnknownLineage {+ // An encrypted/unparseable view column cannot be traced to a base table,+ // so its real lineage — and any base-table masking policy — is unknown.+ // Fail safe and fully mask. The taint rides on the source column, so this+ // one check covers direct refs, expressions, predicates, and subqueries.+ return masker.NewDefaultFullMasker(), &MaskingEvaluation{Algorithm: "Full mask", Context: "Unknown column lineage"}, nil+ }+database := data.getDatabase(sourceColumn.Database)if database == nil {return masker.NewNoneMasker(), nil, nildiff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex bc223c8dfb..46eace6c06 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -102,6 +102,12 @@ type ColumnResource struct {Table string// Column is the normalized column name, it should not be empty.Column string+ // UnknownLineage marks a source whose lineage could not be traced to a base+ // table (e.g. an encrypted/unparseable view whose body is hidden). The masker+ // fully masks any result fed by such a source. Because source columns are+ // unioned as expressions and subqueries are merged, the taint propagates+ // without each merge site having to know about it.+ UnknownLineage bool}// String returns the string format of the column resource.@@ -234,6 +240,10 @@ type PhysicalTable struct {Name string// Columns are the columns of the table.Columns []string+ // UnknownLineage marks a temp table populated (via SELECT ... INTO) from at+ // least one unknown-lineage source. Columns resolved from it inherit the taint+ // so the masker still fully masks data laundered through the temp table.+ UnknownLineage bool}func (p *PhysicalTable) GetTableName() string {@@ -257,11 +267,12 @@ func (p *PhysicalTable) GetQuerySpanResult() []QuerySpanResult {for _, column := range p.Columns {sourceColumnSet := make(SourceColumnSet, 1)sourceColumnSet[ColumnResource{- Server: p.Server,- Database: p.Database,- Schema: p.Schema,- Table: p.Name,- Column: column,+ Server: p.Server,+ Database: p.Database,+ Schema: p.Schema,+ Table: p.Name,+ Column: column,+ UnknownLineage: p.UnknownLineage,}] = trueresult = append(result, QuerySpanResult{Name: column,diff --git a/backend/plugin/parser/tsql/query_span_extractor.go b/backend/plugin/parser/tsql/query_span_extractor.goindex a544007927..acabd275a7 100644--- a/backend/plugin/parser/tsql/query_span_extractor.go+++ b/backend/plugin/parser/tsql/query_span_extractor.go@@ -141,7 +141,40 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabasview := schemaSchema.GetView(viewName)viewColumns, err := q.getColumnsFromCreateView(view.Definition, databaseName, schemaName, viewName)if err != nil {- return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)+ if !errors.Is(err, errNoCreateViewInDefinition) {+ return nil, errors.Wrapf(err, "failed to get columns for view %s.%s.%s", databaseName, schemaName, viewName)+ }+ // The definition has no parseable CREATE VIEW body — typically a+ // SQL Server WITH ENCRYPTION view, whose source the catalog does+ // not expose. Fall back to the synced column metadata (populated+ // from sys.columns regardless of encryption) so column resolution+ // and SQL Editor browsing keep working. Lineage to the base tables+ // is unrecoverable, so each source column is marked UnknownLineage+ // and the masker fully masks anything fed by it (fail-safe), rather+ // than leak data a base-table masking policy would protect.+ cols := view.GetColumns()+ if len(cols) == 0 {+ // No parseable body AND no synced columns: we cannot even+ // enumerate the result shape, so a SELECT * would return+ // columns we have no way to mask. Fail closed.+ return nil, errors.Errorf("cannot resolve columns for view %s.%s.%s: encrypted/unparseable definition with no synced columns", databaseName, schemaName, viewName)+ }+ viewColumns = make([]base.QuerySpanResult, 0, len(cols))+ for _, col := range cols {+ viewColumns = append(viewColumns, base.QuerySpanResult{+ Name: col.GetName(),+ IsPlainField: true,+ SourceColumns: base.SourceColumnSet{+ base.ColumnResource{+ Database: databaseName,+ Schema: schemaName,+ Table: viewName,+ Column: col.GetName(),+ UnknownLineage: true,+ }: true,+ },+ })+ }}return &base.PhysicalView{Database: databaseName,@@ -159,6 +192,12 @@ func (q *querySpanExtractor) tsqlFindTableSchemaByParts(linkedServer, rawDatabas}}+// errNoCreateViewInDefinition signals that a view's stored definition contains+// no parseable CREATE VIEW statement. SQL Server returns only a placeholder+// comment for WITH ENCRYPTION views, which parses to zero statements; callers+// detect this sentinel and fall back to the view's synced column metadata.+var errNoCreateViewInDefinition = errors.New("no CREATE VIEW statement found in definition")+// getColumnsFromCreateView parses a CREATE VIEW definition with omni, extracts// the body's result columns, and applies the optional column alias list.func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDatabaseName string, viewSchemaName string, viewName string) ([]base.QuerySpanResult, error) {@@ -179,7 +218,7 @@ func (q *querySpanExtractor) getColumnsFromCreateView(definition string, viewDat}}if createView == nil {- return nil, errors.Errorf("no CREATE VIEW statement found in definition")+ return nil, errNoCreateViewInDefinition}body, ok := createView.Query.(*ast.SelectStmt)if !ok || body == nil {diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni.go b/backend/plugin/parser/tsql/query_span_extractor_omni.goindex af6d0d0c6b..7206664ddf 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni.go@@ -803,11 +803,12 @@ func (q *omniQuerySpanExtractor) resolveTableVar(tv *ast.TableVarRef, alias striName: c,SourceColumns: base.SourceColumnSet{base.ColumnResource{- Server: temp.Server,- Database: temp.Database,- Schema: temp.Schema,- Table: temp.Name,- Column: c,+ Server: temp.Server,+ Database: temp.Database,+ Schema: temp.Schema,+ Table: temp.Name,+ Column: c,+ UnknownLineage: temp.UnknownLineage,}: true,},IsPlainField: true,@@ -870,16 +871,24 @@ func (q *omniQuerySpanExtractor) populateSelectIntoTempTable(sel *ast.SelectStmtreturn}columns := make([]string, 0, len(results))+ unknownLineage := falsefor i, result := range results {name := result.Nameif name == "" && sel.TargetList != nil && i < len(sel.TargetList.Items) {name = q.sliceName(sel.TargetList.Items[i])}columns = append(columns, name)+ for src := range result.SourceColumns {+ if src.UnknownLineage {+ unknownLineage = true+ break+ }+ }}q.gCtx.TempTables[sel.IntoTable.Object] = &base.PhysicalTable{- Name: sel.IntoTable.Object,- Columns: columns,+ Name: sel.IntoTable.Object,+ Columns: columns,+ UnknownLineage: unknownLineage,}}diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go b/backend/plugin/parser/tsql/query_span_extractor_omni_test.goindex 922d86c6a8..53b5677dd2 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go@@ -14,7 +14,7 @@ import ()// omniTestMetadata is the default mock catalog used by these tests.-// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and a view vw.+// db.dbo has tables t(a,b,c), t1(a,b,c), t2(a,b), and views vw, enc_vw, enc_empty_vw.var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{{Name: "db",@@ -28,6 +28,20 @@ var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{},Views: []*storepb.ViewMetadata{{Name: "vw", Definition: "CREATE VIEW [dbo].[vw] AS SELECT a, b FROM t"},+ // enc_vw mimics a SQL Server WITH ENCRYPTION view: the body is not+ // retrievable, so sync stores a placeholder comment instead of DDL,+ // but the column shape is still synced from sys.columns.+ {+ Name: "enc_vw",+ Definition: "/* Definition of view dbo.enc_vw is encrypted. */",+ Columns: []*storepb.ColumnMetadata{{Name: "id"}, {Name: "secret"}},+ },+ // enc_empty_vw is an encrypted view whose columns were never synced+ // (e.g. sync lacked sys.columns access): browsing must still not error.+ {+ Name: "enc_empty_vw",+ Definition: "/* Definition of view dbo.enc_empty_vw is encrypted. */",+ },},},},@@ -229,6 +243,64 @@ func TestOmniQuerySpan_SupportedShapes(t *testing.T) {}}+// sourcesUnknownLineage reports whether any source column in the set is flagged+// with unknown lineage.+func sourcesUnknownLineage(set base.SourceColumnSet) bool {+ for c := range set {+ if c.UnknownLineage {+ return true+ }+ }+ return false+}++func TestOmniQuerySpan_EncryptedViewUnknownLineage(t *testing.T) {+ // Direct references to an encrypted view's columns resolve from synced+ // metadata, but their lineage is unrecoverable, so each source column must be+ // flagged UnknownLineage for the masker to fully mask it.+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT id, secret FROM enc_vw")+ require.NoError(t, err)+ require.Len(t, span.Results, 2)+ for _, r := range span.Results {+ require.Truef(t, sourcesUnknownLineage(r.SourceColumns), "result %q should have an unknown-lineage source", r.Name)+ }++ // The taint must survive an expression that merges sources; otherwise+ // `secret + 'x'` over an encrypted column would reach the masker untainted+ // and be returned unmasked.+ q2 := newOmniTestExtractor(t, "db")+ span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret + 'x' AS e FROM enc_vw")+ require.NoError(t, err)+ require.Len(t, span2.Results, 1)+ require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "expression over an encrypted column should stay unknown-lineage")++ // Control: a normal view resolves to real base-table lineage, untainted.+ q3 := newOmniTestExtractor(t, "db")+ span3, err := q3.getOmniQuerySpan(context.Background(), "SELECT a, b FROM vw")+ require.NoError(t, err)+ require.NotEmpty(t, span3.Results)+ for _, r := range span3.Results {+ require.Falsef(t, sourcesUnknownLineage(r.SourceColumns), "result %q from normal view should have known lineage", r.Name)+ }++ // A predicate-only reference to an encrypted column must also carry the taint,+ // so error redaction over PredicateColumns covers `WHERE secret = ...`.+ q4 := newOmniTestExtractor(t, "db")+ span4, err := q4.getOmniQuerySpan(context.Background(), "SELECT 1 FROM enc_vw WHERE secret = 'x'")+ require.NoError(t, err)+ require.True(t, sourcesUnknownLineage(span4.PredicateColumns), "predicate over an encrypted column should be unknown-lineage")+}++// An encrypted view with no synced columns can neither be parsed nor have its+// result shape enumerated, so it must fail closed rather than resolve to columns+// that cannot be masked.+func TestOmniQuerySpan_EncryptedViewNoColumnsFailsClosed(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT * FROM enc_empty_vw")+ require.Error(t, err)+}+func TestOmniQuerySpan_NotFound(t *testing.T) {q := newOmniTestExtractor(t, "db")span, err := q.getOmniQuerySpan(context.Background(), "SELECT a FROM no_such_table")@@ -404,6 +476,30 @@ func TestOmniQuerySpan_TempTableDefinitions(t *testing.T) {require.Equal(t, "name", span2.Results[1].Name)}+// SELECT ... INTO #tmp from an encrypted view must carry the unknown-lineage+// taint into the temp table, otherwise a later query of #tmp would launder the+// data past the masker unmasked.+func TestOmniQuerySpan_SelectIntoEncryptedViewKeepsTaint(t *testing.T) {+ getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)+ gCtx := base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ TempTables: make(map[string]*base.PhysicalTable),+ }++ q1 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)+ _, err := q1.getOmniQuerySpan(context.Background(), "SELECT secret INTO #tmp FROM enc_vw")+ require.NoError(t, err)+ require.Contains(t, gCtx.TempTables, "#tmp")+ require.True(t, gCtx.TempTables["#tmp"].UnknownLineage, "temp table from an encrypted view should be flagged unknown-lineage")++ q2 := newOmniQuerySpanExtractor("db", "dbo", gCtx, true)+ span2, err := q2.getOmniQuerySpan(context.Background(), "SELECT secret FROM #tmp")+ require.NoError(t, err)+ require.Len(t, span2.Results, 1)+ require.True(t, sourcesUnknownLineage(span2.Results[0].SourceColumns), "column from a tainted temp table should stay unknown-lineage")+}+func TestOmniQuerySpan_TempTableDefinitionsCaseInsensitive(t *testing.T) {getter, lister := buildMockDatabaseMetadataGetter(omniTestMetadata)gCtx := base.GetQuerySpanContext{