Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(trino): catalog-aware masking lineage — wire GetQuerySpanWithCatalog (BYT-9674..9680) (#20565)
h3n4l8c17ad04e7e9
diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.goindex ae5f7770e1..ee0ae8037e 100644--- a/backend/plugin/parser/trino/completion.go+++ b/backend/plugin/parser/trino/completion.go@@ -10,6 +10,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model")func init() {@@ -88,28 +89,7 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, stif err != nil || meta == nil {continue}- database := cat.EnsureCatalog(norm)- for _, schemaName := range meta.ListSchemaNames() {- schemaMeta := meta.GetSchemaMetadata(schemaName)- if schemaMeta == nil {- continue- }- sc := database.EnsureSchema(catalog.Normalize(schemaName))- for _, tableName := range schemaMeta.ListTableNames() {- tableMeta := schemaMeta.GetTable(tableName)- if tableMeta == nil {- continue- }- sc.AddTable(catalog.Normalize(tableName), columnsOf(tableMeta.GetProto().GetColumns())...)- }- for _, viewName := range schemaMeta.ListViewNames() {- viewMeta := schemaMeta.GetView(viewName)- if viewMeta == nil {- continue- }- sc.AddView(catalog.Normalize(viewName), columnsOf(viewMeta.GetColumns())...)- }- }+ loadCatalogMetadata(cat, norm, meta)}if cCtx.DefaultDatabase != "" {@@ -121,6 +101,43 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, streturn cat}+// loadCatalogMetadata copies one database's schemas, tables and views (with+// their column lists and view definitions) into the omni catalog under the+// given normalized catalog name, returning the view definitions it loaded.+// Views carry their defining query so omni's analysis can resolve lineage+// through them (GetQuerySpanWithCatalog); an empty definition leaves the view+// opaque. Shared by the completion and query-span catalog builders.+func loadCatalogMetadata(cat *catalog.Catalog, norm string, meta *model.DatabaseMetadata) []string {+ var definitions []string+ database := cat.EnsureCatalog(norm)+ for _, schemaName := range meta.ListSchemaNames() {+ schemaMeta := meta.GetSchemaMetadata(schemaName)+ if schemaMeta == nil {+ continue+ }+ sc := database.EnsureSchema(catalog.Normalize(schemaName))+ for _, tableName := range schemaMeta.ListTableNames() {+ tableMeta := schemaMeta.GetTable(tableName)+ if tableMeta == nil {+ continue+ }+ sc.AddTable(catalog.Normalize(tableName), columnsOf(tableMeta.GetProto().GetColumns())...)+ }+ for _, viewName := range schemaMeta.ListViewNames() {+ viewMeta := schemaMeta.GetView(viewName)+ if viewMeta == nil {+ continue+ }+ v := sc.AddView(catalog.Normalize(viewName), columnsOf(viewMeta.GetColumns())...)+ v.Definition = viewMeta.GetDefinition()+ if def := viewMeta.GetDefinition(); def != "" {+ definitions = append(definitions, def)+ }+ }+ }+ return definitions+}+// catalogNeeded reports whether the current completion statement needs this// catalog's full schema/table/column metadata loaded: true for the session// default catalog and for any catalog whose name appears in the statement textdiff --git a/backend/plugin/parser/trino/query_span_extractor.go b/backend/plugin/parser/trino/query_span_extractor.goindex 98c6104715..3c57fcc3c8 100644--- a/backend/plugin/parser/trino/query_span_extractor.go+++ b/backend/plugin/parser/trino/query_span_extractor.go@@ -6,6 +6,7 @@ import ("github.com/bytebase/omni/trino/analysis""github.com/bytebase/omni/trino/ast"+ "github.com/bytebase/omni/trino/catalog""github.com/bytebase/omni/trino/parser""github.com/pkg/errors"@@ -119,7 +120,7 @@ func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string)}}- omniSpan, err := analysis.GetQuerySpan(spanInput)+ omniSpan, err := analysis.GetQuerySpanWithCatalog(spanInput, q.buildSpanCatalog(ctx, spanInput))if err != nil {return nil, err}@@ -167,6 +168,78 @@ func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string)return span, nil}+// buildSpanCatalog constructs the omni Trino catalog the query-span analysis+// resolves against: views carry their defining query (so lineage through a view+// reaches the underlying base-table columns — masking config only attaches to+// table columns), and tables carry their column lists (so omni expands+// SELECT * to the exact projection). The session context is the extractor's+// default catalog/schema (with the same "public" fallback as+// resolveCatalogSchema, so view resolution and resource keys agree).+//+// Like the completion catalog, every catalog name is registered cheaply but+// full metadata is loaded only for the catalogs the statement needs (the+// default one and any named in the statement). Metadata fetches go through the+// extractor's cache, so the later expandTablesToColumns lookups reuse them.+// Returns nil — omni then behaves exactly catalog-less — when metadata is+// unavailable (e.g. tests without a getter).+func (q *querySpanExtractor) buildSpanCatalog(ctx context.Context, statement string) *catalog.Catalog {+ q.ctx = ctx+ if q.gCtx.GetDatabaseMetadataFunc == nil || q.gCtx.ListDatabaseNamesFunc == nil {+ return nil+ }+ names, err := q.gCtx.ListDatabaseNamesFunc(ctx, q.gCtx.InstanceID)+ if err != nil || len(names) == 0 {+ return nil+ }++ defaultDB := catalog.Normalize(q.defaultDatabase)+ cat := catalog.New()+ for _, dbName := range names {+ cat.EnsureCatalog(catalog.Normalize(dbName))+ }++ // Load catalogs transitively: the statement names some catalogs; the view+ // definitions loaded with them may name FURTHER catalogs (a view in the+ // default catalog defined over another catalog's table), which omni needs+ // loaded to resolve lineage through the view. Each round scans the text+ // corpus (statement + every loaded view definition) and loads any+ // newly-referenced catalog; it terminates because each catalog loads at+ // most once.+ corpus := strings.ToLower(statement)+ loaded := make(map[string]bool, len(names))+ for {+ progressed := false+ for _, dbName := range names {+ norm := catalog.Normalize(dbName)+ if loaded[norm] || !catalogNeeded(norm, defaultDB, corpus) {+ continue+ }+ loaded[norm] = true+ progressed = true+ meta, err := q.getDatabaseMetadata(dbName)+ if err != nil || meta == nil {+ continue+ }+ for _, def := range loadCatalogMetadata(cat, norm, meta) {+ corpus += "\n" + strings.ToLower(def)+ }+ }+ if !progressed {+ break+ }+ }++ if q.defaultDatabase != "" {+ cat.SetCurrentCatalog(defaultDB)+ }+ schema := q.defaultSchema+ if schema == "" {+ schema = defaultTrinoSchema+ }+ cat.SetCurrentSchema(catalog.Normalize(schema))+ return cat+}+// toTableResources converts omni AccessTables into table-level// base.ColumnResource keys (Column empty), applying default catalog/schema.func (q *querySpanExtractor) toTableResources(span *analysis.QuerySpan) base.SourceColumnSet {@@ -333,7 +406,15 @@ func (q *querySpanExtractor) buildResults(span *analysis.QuerySpan, fullSourceCoresults = append(results, base.QuerySpanResult{Name: r.Name,SourceColumns: sourceColumns,- IsPlainField: len(r.SourceColumns) == 1,+ // Plainness keys on the MAPPED physical set, not omni's raw ref+ // count: the catalog-aware resolver additively restates a plain+ // column as written + catalog-qualified refs, which dedupe back to+ // the one physical column here. Known drift: an expression over one+ // column repeated (phone || phone) also maps to a single physical+ // column and reads as plain — omni's resolver dedups refs, so the+ // distinction is unrecoverable here; inert for Trino, which is not+ // in EngineSupportQuerySpanPlainField.+ IsPlainField: len(sourceColumns) == 1,})}return resultsdiff --git a/backend/plugin/parser/trino/query_span_lineage_test.go b/backend/plugin/parser/trino/query_span_lineage_test.gonew file mode 100644index 0000000000..89bf4bc44b--- /dev/null+++ b/backend/plugin/parser/trino/query_span_lineage_test.go@@ -0,0 +1,120 @@+package trino++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// These tests pin the seven under-masking vectors from the Trino masking audit+// (BYT-9674..BYT-9680, sub-issues of BYT-9142) at the consumer level: each+// result column backed by a sensitive base column must carry that base column+// in its lineage, so the result masker attaches the configured masker instead+// of falling through to NoneMasker. The lineage resolution itself lives in+// omni's trino/analysis (catalog-aware since GetQuerySpanWithCatalog); this+// exercises the full extractor path metadata -> catalog -> omni -> resource+// mapping. The view vectors are covered in query_span_view_test.go.++func lineageAuditMetadata() *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "catalog1",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {+ Name: "customer",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "integer"},+ {Name: "phone", Type: "varchar"},+ {Name: "name", Type: "varchar"},+ {Name: "phones", Type: "array(varchar)"},+ },+ },+ },+ },+ },+ }+}++func lineageAuditSpan(t *testing.T, sql string) *base.QuerySpan {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{lineageAuditMetadata()})+ gCtx := base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ Engine: storepb.Engine_TRINO,+ }+ extractor := newQuerySpanExtractor("catalog1", "public", gCtx, false)+ span, err := extractor.getQuerySpan(context.Background(), sql)+ require.NoError(t, err)+ return span+}++func auditCol(col string) base.ColumnResource {+ return base.ColumnResource{Database: "catalog1", Schema: "public", Table: "customer", Column: col}+}++// BYT-9674: a column projected through a derived table and re-aliased.+func TestLineageAudit_DerivedTableAlias(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT d.x FROM (SELECT phone AS x FROM public.customer) d")+ srcs := resultSources(t, span, "x")+ require.True(t, srcs[auditCol("phone")], "x should reach customer.phone; got %+v", srcs)+}++// BYT-9675: a column projected through a CTE and re-aliased.+func TestLineageAudit_CTEAlias(t *testing.T) {+ span := lineageAuditSpan(t, "WITH w AS (SELECT phone AS pp FROM public.customer) SELECT pp FROM w")+ srcs := resultSources(t, span, "pp")+ require.True(t, srcs[auditCol("phone")], "pp should reach customer.phone; got %+v", srcs)+}++// BYT-9676: a scalar subquery used as a SELECT value.+func TestLineageAudit_ScalarSubquery(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT (SELECT phone FROM public.customer LIMIT 1) AS sp FROM public.customer")+ srcs := resultSources(t, span, "sp")+ require.True(t, srcs[auditCol("phone")], "sp should reach customer.phone; got %+v", srcs)+}++// BYT-9677: a set operation's output column carries every arm's lineage, not+// just the first arm's.+func TestLineageAudit_SetOpArmMerge(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT name FROM public.customer UNION SELECT phone FROM public.customer")+ srcs := resultSources(t, span, "name")+ require.True(t, srcs[auditCol("name")], "left arm lineage expected; got %+v", srcs)+ require.True(t, srcs[auditCol("phone")], "right arm (sensitive) lineage must be merged; got %+v", srcs)+}++// BYT-9678: SELECT * over a derived table expands to the derived projection+// (width and order), not the base table's column set — the original+// positional-masker repro: the span used to report 4 columns where Trino+// returns 2, sliding phone under id's (None) masker.+func TestLineageAudit_StarOverDerived(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT * FROM (SELECT phone, name FROM public.customer) d")+ require.Len(t, span.Results, 2, "span must match the executed width [phone name]; got %+v", span.Results)+ require.Equal(t, "phone", span.Results[0].Name)+ require.Equal(t, "name", span.Results[1].Name)+ require.True(t, span.Results[0].SourceColumns[auditCol("phone")], "Results[0] -> customer.phone; got %+v", span.Results[0].SourceColumns)+ require.True(t, span.Results[1].SourceColumns[auditCol("name")], "Results[1] -> customer.name; got %+v", span.Results[1].SourceColumns)+}++// BYT-9680: a column produced by UNNEST of a sensitive array column.+func TestLineageAudit_UnnestAlias(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT t.p FROM public.customer CROSS JOIN UNNEST(phones) AS t(p)")+ srcs := resultSources(t, span, "p")+ require.True(t, srcs[auditCol("phones")], "p should reach customer.phones; got %+v", srcs)+}++// Star over a base table still expands positionally against metadata (sanity+// that the catalog-aware path keeps the executed order).+func TestLineageAudit_StarOverBase(t *testing.T) {+ span := lineageAuditSpan(t, "SELECT * FROM public.customer")+ require.Len(t, span.Results, 4)+ require.Equal(t, "id", span.Results[0].Name)+ require.Equal(t, "phone", span.Results[1].Name)+ require.True(t, span.Results[1].SourceColumns[auditCol("phone")], "Results[1] -> customer.phone; got %+v", span.Results[1].SourceColumns)+}diff --git a/backend/plugin/parser/trino/query_span_view_test.go b/backend/plugin/parser/trino/query_span_view_test.gonew file mode 100644index 0000000000..0dff5b5d71--- /dev/null+++ b/backend/plugin/parser/trino/query_span_view_test.go@@ -0,0 +1,221 @@+package trino++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// viewLineageMetadata builds a catalog with a base table customer(id, phone,+// name), a pass-through view customer_v, a renaming view cust_masked+// (phone AS ph), and a view-over-view v2.+func viewLineageMetadata() *storepb.DatabaseSchemaMetadata {+ return &storepb.DatabaseSchemaMetadata{+ Name: "catalog1",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {+ Name: "customer",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "integer"},+ {Name: "phone", Type: "varchar"},+ {Name: "name", Type: "varchar"},+ },+ },+ },+ Views: []*storepb.ViewMetadata{+ {+ Name: "customer_v",+ Definition: "SELECT id, phone, name FROM customer",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id"}, {Name: "phone"}, {Name: "name"},+ },+ },+ {+ Name: "cust_masked",+ Definition: "SELECT phone AS ph FROM customer",+ Columns: []*storepb.ColumnMetadata{{Name: "ph"}},+ },+ {+ Name: "v2",+ Definition: "SELECT phone FROM customer_v",+ Columns: []*storepb.ColumnMetadata{{Name: "phone"}},+ },+ {+ // Explicit view column list renames the projection: the+ // definition outputs "phone" but the view column is "ph".+ Name: "renamed_v",+ Definition: "SELECT phone FROM customer",+ Columns: []*storepb.ColumnMetadata{{Name: "ph"}},+ },+ {+ // A composed view column derives from two base columns.+ Name: "comp_v",+ Definition: "SELECT phone || name AS token FROM customer",+ Columns: []*storepb.ColumnMetadata{{Name: "token"}},+ },+ },+ },+ },+ }+}++func viewLineageSpan(t *testing.T, sql string) *base.QuerySpan {+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{viewLineageMetadata()})+ gCtx := base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ Engine: storepb.Engine_TRINO,+ }+ extractor := newQuerySpanExtractor("catalog1", "public", gCtx, false)+ span, err := extractor.getQuerySpan(context.Background(), sql)+ require.NoError(t, err)+ return span+}++func resultSources(t *testing.T, span *base.QuerySpan, name string) base.SourceColumnSet {+ t.Helper()+ for _, r := range span.Results {+ if r.Name == name {+ return r.SourceColumns+ }+ }+ t.Fatalf("no result column named %q in %+v", name, span.Results)+ return nil+}++func customerCol(col string) base.ColumnResource {+ return base.ColumnResource{Database: "catalog1", Schema: "public", Table: "customer", Column: col}+}++// TestQuerySpan_ViewColumnResolvesToBase covers BYT-9679: a column selected+// through a view must carry the underlying base-table column (where masking+// config lives) in its lineage. omni's resolution is additive — the view column+// ref is retained beside the base column — which is safe: the view column has+// no masking catalog and contributes nothing, while the base column's masker+// applies.+func TestQuerySpan_ViewColumnResolvesToBase(t *testing.T) {+ span := viewLineageSpan(t, "SELECT phone FROM customer_v")+ srcs := resultSources(t, span, "phone")+ require.True(t, srcs[customerCol("phone")], "phone should resolve to base customer.phone; got %+v", srcs)+}++// TestQuerySpan_ViewStarResolvesToBase resolves every column of SELECT * over a+// view to its base columns.+func TestQuerySpan_ViewStarResolvesToBase(t *testing.T) {+ span := viewLineageSpan(t, "SELECT * FROM customer_v")+ require.True(t, resultSources(t, span, "phone")[customerCol("phone")], "SELECT * phone should resolve to customer.phone")+ require.True(t, resultSources(t, span, "id")[customerCol("id")], "SELECT * id should resolve to customer.id")+ require.True(t, resultSources(t, span, "name")[customerCol("name")], "SELECT * name should resolve to customer.name")+}++// TestQuerySpan_RenamingViewResolvesToBase resolves a view that renames the+// projected column (phone AS ph): the outer ph must map to base customer.phone.+func TestQuerySpan_RenamingViewResolvesToBase(t *testing.T) {+ span := viewLineageSpan(t, "SELECT ph FROM cust_masked")+ srcs := resultSources(t, span, "ph")+ require.True(t, srcs[customerCol("phone")], "ph should resolve to base customer.phone; got %+v", srcs)+}++// TestQuerySpan_ViewOverViewResolvesToBase resolves a view defined over another+// view transitively to the base column.+func TestQuerySpan_ViewOverViewResolvesToBase(t *testing.T) {+ span := viewLineageSpan(t, "SELECT phone FROM v2")+ srcs := resultSources(t, span, "phone")+ require.True(t, srcs[customerCol("phone")], "phone should resolve through v2 -> customer_v -> customer.phone; got %+v", srcs)+}++// TestQuerySpan_ViewExplicitColumnListResolvesToBase covers a view whose column+// list renames the projection (CREATE VIEW renamed_v (ph) AS SELECT phone ...):+// the outer ph must map positionally to base customer.phone even though the+// definition's output name is "phone".+func TestQuerySpan_ViewExplicitColumnListResolvesToBase(t *testing.T) {+ span := viewLineageSpan(t, "SELECT ph FROM renamed_v")+ srcs := resultSources(t, span, "ph")+ require.True(t, srcs[customerCol("phone")], "ph should resolve positionally to base customer.phone; got %+v", srcs)+}++// TestQuerySpan_ComposedViewColumnNotPlain covers a view column composed from two+// base columns: it must resolve to both, and must not stay a plain field.+func TestQuerySpan_ComposedViewColumnNotPlain(t *testing.T) {+ span := viewLineageSpan(t, "SELECT token FROM comp_v")+ for _, r := range span.Results {+ if r.Name != "token" {+ continue+ }+ require.True(t, r.SourceColumns[customerCol("phone")], "token should include customer.phone; got %+v", r.SourceColumns)+ require.True(t, r.SourceColumns[customerCol("name")], "token should include customer.name; got %+v", r.SourceColumns)+ require.False(t, r.IsPlainField, "a composed view column must not be a plain field")+ return+ }+ t.Fatalf("no result column named token in %+v", span.Results)+}++// TestQuerySpan_CrossCatalogViewResolvesToBase covers a view defined over a+// DIFFERENT catalog's table (with a star body, so resolution needs that+// catalog's column metadata): the outer statement never names catalog2, so the+// span-catalog builder must follow the view definition transitively to load it.+func TestQuerySpan_CrossCatalogViewResolvesToBase(t *testing.T) {+ cat1 := &storepb.DatabaseSchemaMetadata{+ Name: "catalog1",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Views: []*storepb.ViewMetadata{+ {+ Name: "xcat_v",+ Definition: "SELECT * FROM catalog2.public.customer",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id"}, {Name: "phone"}, {Name: "name"},+ },+ },+ },+ },+ },+ }+ cat2 := &storepb.DatabaseSchemaMetadata{+ Name: "catalog2",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {+ Name: "customer",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "integer"},+ {Name: "phone", Type: "varchar"},+ {Name: "name", Type: "varchar"},+ },+ },+ },+ },+ },+ }+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{cat1, cat2})+ gCtx := base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ Engine: storepb.Engine_TRINO,+ }+ extractor := newQuerySpanExtractor("catalog1", "public", gCtx, false)+ span, err := extractor.getQuerySpan(context.Background(), "SELECT phone FROM xcat_v")+ require.NoError(t, err)+ srcs := resultSources(t, span, "phone")+ want := base.ColumnResource{Database: "catalog2", Schema: "public", Table: "customer", Column: "phone"}+ require.True(t, srcs[want], "phone should resolve through xcat_v to catalog2.public.customer.phone; got %+v", srcs)+}++// TestQuerySpan_BaseTableUnaffectedByViewResolution guards that a plain base+// table query is unchanged by the view post-pass.+func TestQuerySpan_BaseTableUnaffectedByViewResolution(t *testing.T) {+ span := viewLineageSpan(t, "SELECT phone FROM customer")+ srcs := resultSources(t, span, "phone")+ require.True(t, srcs[customerCol("phone")], "base customer.phone expected; got %+v", srcs)+ require.Len(t, srcs, 1, "base-table lineage should be exactly customer.phone; got %+v", srcs)+}