Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(tsql): resolve pseudo-columns in query span; suggest $action in completion (#20754)
rebelice07b539afe8b7
diff --git a/backend/plugin/parser/tsql/completion.go b/backend/plugin/parser/tsql/completion.goindex f6551b6968..a0f7591daf 100644--- a/backend/plugin/parser/tsql/completion.go+++ b/backend/plugin/parser/tsql/completion.go@@ -777,6 +777,13 @@ func (c *Completer) convertCandidates(candidates *mssqlparser.CandidateSet) ([]bfor _, context := range completionContexts {c.insertObjectCandidates(context, databaseEntries, schemaEntries, tableEntries, viewEntries, sequenceEntries)}+ case "pseudocolumn_action":+ // OUTPUT clause position: suggest the $action pseudo-column+ // (MERGE OUTPUT returns 'INSERT'/'UPDATE'/'DELETE' per row).+ columnEntries.Insert(base.Candidate{+ Type: base.CandidateTypeColumn,+ Text: "$action",+ })case "columnref":completionContexts := c.determineColumnNameContext()for _, context := range completionContexts {diff --git a/backend/plugin/parser/tsql/completion_pseudo_column_test.go b/backend/plugin/parser/tsql/completion_pseudo_column_test.gonew file mode 100644index 0000000000..a628720abd--- /dev/null+++ b/backend/plugin/parser/tsql/completion_pseudo_column_test.go@@ -0,0 +1,30 @@+package tsql++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// TestCompletionPseudoColumnAction pins that the OUTPUT-clause position+// surfaces the $action pseudo-column: omni emits the pseudocolumn_action rule+// there, which previously fell through Bytebase's rule switch silently.+func TestCompletionPseudoColumnAction(t *testing.T) {+ statement, caretLine, caretPosition := getCaretPosition(+ "MERGE INTO t AS d USING s ON d.k = s.k WHEN MATCHED THEN UPDATE SET d.v = s.v OUTPUT |")+ getter, lister := buildMockDatabaseMetadataGetterLister()+ results, err := Completion(context.Background(), base.CompletionContext{+ Scene: base.SceneTypeAll,+ DefaultDatabase: "Company",+ Metadata: getter,+ ListDatabaseNames: lister,+ }, statement, caretLine, caretPosition)+ require.NoError(t, err)+ require.Contains(t, results, base.Candidate{+ Type: base.CandidateTypeColumn,+ Text: "$action",+ })+}diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni.go b/backend/plugin/parser/tsql/query_span_extractor_omni.goindex af6d0d0c6b..0f75539b56 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni.go@@ -192,6 +192,13 @@ func (q *omniQuerySpanExtractor) collectPredicatesInScope(expr ast.Node) error {}switch v := expr.(type) {case *ast.ColumnRef:+ if r, handled, err := q.resolvePseudoColumnRef(v); handled {+ if err != nil {+ return errors.Wrapf(err, "failed to resolve predicate column %q", v.Column)+ }+ q.predicateColumns, _ = base.MergeSourceColumnSet(q.predicateColumns, r.SourceColumns)+ return nil+ }r, err := q.tsqlIsFieldSensitive(v.Database, v.Schema, v.Table, v.Column)if err != nil {return errors.Wrapf(err, "failed to resolve predicate column %q", v.Column)@@ -1001,6 +1008,12 @@ func (q *omniQuerySpanExtractor) resolveExpressionNode(n ast.Node) (base.QuerySp}switch v := n.(type) {case *ast.ColumnRef:+ if r, handled, err := q.resolvePseudoColumnRef(v); handled {+ if err != nil {+ return base.QuerySpanResult{}, errors.Wrapf(err, "failed to resolve column %q", v.Column)+ }+ return r, nil+ }r, err := q.tsqlIsFieldSensitive(v.Database, v.Schema, v.Table, v.Column)if err != nil {return base.QuerySpanResult{}, errors.Wrapf(err, "failed to resolve column %q", v.Column)diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go b/backend/plugin/parser/tsql/query_span_extractor_omni_test.goindex 922d86c6a8..be55586fd1 100644--- a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go+++ b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go@@ -25,6 +25,10 @@ var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{{Name: "t", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}, {Name: "c"}}},{Name: "t1", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}, {Name: "c"}}},{Name: "t2", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}}},+ {Name: "ident_t", Columns: []*storepb.ColumnMetadata{{Name: "id", IsIdentity: true}, {Name: "payload"}}},+ {Name: "ident_t2", Columns: []*storepb.ColumnMetadata{{Name: "seq", IsIdentity: true}, {Name: "v"}}},+ {Name: "bare_edge"},+ {Name: "weird", Columns: []*storepb.ColumnMetadata{{Name: "$node_id"}, {Name: "IDENTITYCOL"}}},},Views: []*storepb.ViewMetadata{{Name: "vw", Definition: "CREATE VIEW [dbo].[vw] AS SELECT a, b FROM t"},diff --git a/backend/plugin/parser/tsql/query_span_pseudo_column_omni.go b/backend/plugin/parser/tsql/query_span_pseudo_column_omni.gonew file mode 100644index 0000000000..97332c8d7a--- /dev/null+++ b/backend/plugin/parser/tsql/query_span_pseudo_column_omni.go@@ -0,0 +1,211 @@+package tsql++import (+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"++ "github.com/bytebase/omni/mssql/ast"+)++// pseudoColumnKind classifies T-SQL pseudo-column references the omni parser+// accepts as ColumnRef nodes.+type pseudoColumnKind int++const (+ pseudoColumnNone pseudoColumnKind = iota+ // pseudoColumnIdentity is $IDENTITY / IDENTITYCOL: the engine resolves it+ // to the table's single IDENTITY-property column at bind time.+ pseudoColumnIdentity+ // pseudoColumnGraph is $node_id / $edge_id / $from_id / $to_id on graph+ // tables. The backing engine columns are auto-generated and not present in+ // the synced catalog.+ pseudoColumnGraph+)++func classifyPseudoColumn(name string) pseudoColumnKind {+ switch strings.ToLower(name) {+ case "$identity", "identitycol":+ return pseudoColumnIdentity+ case "$node_id", "$edge_id", "$from_id", "$to_id":+ return pseudoColumnGraph+ default:+ // $ROWGUID / ROWGUIDCOL deliberately unhandled: the synced catalog has+ // no rowguid flag yet, so they keep today's fail-closed resolution+ // error until metadata support lands.+ return pseudoColumnNone+ }+}++// resolvePseudoColumnRef maps a pseudo-column reference to real column+// lineage. handled is false when v is not a (supported) pseudo-column and the+// ordinary resolution path should run.+//+// - $IDENTITY / IDENTITYCOL map precisely to the in-scope table's column+// with ColumnMetadata.IsIdentity, so masking rules on the identity column+// apply to the pseudo-column reference too.+// - Graph pseudo-columns get table-level lineage (the union of the table's+// columns): $from_id/$to_id encode the identity of referenced node rows,+// so empty lineage would fail open in scenarios where relationships+// themselves are sensitive; over-masking is the safer direction.+//+// Both kinds require the reference to bind to exactly one candidate table in+// scope — the engine reports ambiguity errors itself, so mirroring that is+// fail-closed.+func (q *omniQuerySpanExtractor) resolvePseudoColumnRef(v *ast.ColumnRef) (base.QuerySpanResult, bool, error) {+ kind := classifyPseudoColumn(v.Column)+ if kind == pseudoColumnNone {+ return base.QuerySpanResult{}, false, nil+ }+ // Delimited identifiers are never pseudo-columns in T-SQL: [IDENTITYCOL]+ // and [$node_id] reference real columns with those names. omni strips the+ // delimiters from ColumnRef.Column, so check the source text — a+ // delimited column segment makes the ref end with ']' or '"', which a+ // bare pseudo-column or keyword never does.+ if q.columnSegmentIsDelimited(v) {+ return base.QuerySpanResult{}, false, nil+ }++ sources := q.pseudoColumnCandidateSources(v)+ switch kind {+ case pseudoColumnIdentity:+ r, err := q.resolveIdentityPseudoColumn(v, sources)+ return r, true, err+ case pseudoColumnGraph:+ r, err := q.resolveGraphPseudoColumn(v, sources)+ return r, true, err+ default:+ return base.QuerySpanResult{}, false, nil+ }+}++// columnSegmentIsDelimited reports whether the reference's column segment is+// bracket- or quote-delimited in the original source.+func (q *omniQuerySpanExtractor) columnSegmentIsDelimited(v *ast.ColumnRef) bool {+ end := v.Loc.End+ if end <= 0 || end > len(q.source) {+ return false+ }+ last := q.source[end-1]+ return last == ']' || last == '"'+}++// pseudoColumnCandidateSources returns the in-scope table sources the+// reference may bind to, filtered by the optional table qualifier+// (t.$IDENTITY binds within t only). Inner scope is searched before outer,+// matching tsqlIsFieldSensitive.+func (q *omniQuerySpanExtractor) pseudoColumnCandidateSources(v *ast.ColumnRef) []base.TableSource {+ matches := func(ts base.TableSource) bool {+ if v.Table != "" && !q.isIdentifierEqual(v.Table, ts.GetTableName()) {+ return false+ }+ if v.Schema != "" && !q.isIdentifierEqual(v.Schema, ts.GetSchemaName()) {+ return false+ }+ if v.Database != "" && !q.isIdentifierEqual(v.Database, ts.GetDatabaseName()) {+ return false+ }+ return true+ }+ var result []base.TableSource+ for _, ts := range q.tableSourcesFrom {+ if matches(ts) {+ result = append(result, ts)+ }+ }+ if len(result) > 0 {+ return result+ }+ for i := len(q.outerTableSources) - 1; i >= 0; i-- {+ if matches(q.outerTableSources[i]) {+ result = append(result, q.outerTableSources[i])+ }+ }+ return result+}++func (q *omniQuerySpanExtractor) resolveIdentityPseudoColumn(v *ast.ColumnRef, sources []base.TableSource) (base.QuerySpanResult, error) {+ type match struct {+ column base.QuerySpanResult+ }+ var matches []match+ for _, ts := range sources {+ for _, column := range ts.GetQuerySpanResult() {+ if q.isIdentityColumnResult(column) {+ matches = append(matches, match{column: column})+ break // a table has at most one identity column+ }+ }+ }+ switch len(matches) {+ case 0:+ return base.QuerySpanResult{}, errors.Errorf("no identity column found in scope for %q", v.Column)+ case 1:+ r := matches[0].column+ r.IsPlainField = true+ return r, nil+ default:+ return base.QuerySpanResult{}, errors.Errorf("ambiguous %q: multiple tables in scope have an identity column", v.Column)+ }+}++// isIdentityColumnResult reports whether a table-source column is backed by+// exactly one physical column whose metadata has IsIdentity. The lookup goes+// through the column's source resource rather than the table source itself so+// aliased tables (wrapped as PseudoTable) resolve too.+func (q *omniQuerySpanExtractor) isIdentityColumnResult(column base.QuerySpanResult) bool {+ if !column.IsPlainField || len(column.SourceColumns) != 1 {+ return false+ }+ for resource := range column.SourceColumns {+ if resource.Database == "" || resource.Table == "" || resource.Column == "" {+ return false+ }+ _, databaseMeta, err := q.gCtx.GetDatabaseMetadataFunc(q.ctx, q.gCtx.InstanceID, resource.Database)+ if err != nil || databaseMeta == nil {+ return false+ }+ schemaMeta := databaseMeta.GetSchemaMetadata(resource.Schema)+ if schemaMeta == nil {+ return false+ }+ tableMeta := schemaMeta.GetTable(resource.Table)+ if tableMeta == nil {+ return false+ }+ for _, columnMeta := range tableMeta.GetProto().GetColumns() {+ if q.isIdentifierEqual(columnMeta.Name, resource.Column) {+ return columnMeta.IsIdentity+ }+ }+ }+ return false+}++func (*omniQuerySpanExtractor) resolveGraphPseudoColumn(v *ast.ColumnRef, sources []base.TableSource) (base.QuerySpanResult, error) {+ if len(sources) == 0 {+ return base.QuerySpanResult{}, errors.Errorf("no table in scope for graph pseudo-column %q", v.Column)+ }+ if len(sources) > 1 && v.Table == "" {+ return base.QuerySpanResult{}, errors.Errorf("ambiguous graph pseudo-column %q: qualify it with the table name", v.Column)+ }+ sourceColumns := make(base.SourceColumnSet)+ for _, column := range sources[0].GetQuerySpanResult() {+ sourceColumns, _ = base.MergeSourceColumnSet(sourceColumns, column.SourceColumns)+ }+ // Attribute-less graph tables (CREATE TABLE ... AS EDGE with no+ // user-defined columns) contribute no catalog columns, so the union is+ // empty — and empty SourceColumns downstream means NoneMasker, the exact+ // fail-open this table-level lineage is meant to prevent. Fail closed: a+ // table-only resource would not match column-level masking rules either.+ if len(sourceColumns) == 0 {+ return base.QuerySpanResult{}, errors.Errorf("cannot derive lineage for graph pseudo-column %q: table has no columns in the catalog", v.Column)+ }+ return base.QuerySpanResult{+ Name: v.Column,+ SourceColumns: sourceColumns,+ IsPlainField: false,+ }, nil+}diff --git a/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.go b/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.gonew file mode 100644index 0000000000..de52d99f41--- /dev/null+++ b/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.go@@ -0,0 +1,160 @@+package tsql++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// TestOmniQuerySpanPseudoColumns covers the downstream mapping for T-SQL+// pseudo-columns the parser accepts since the $-token omni bump:+// - $IDENTITY / IDENTITYCOL map precisely to the in-scope table's+// ColumnMetadata.IsIdentity column, so masking rules on the identity+// column follow the pseudo-column reference.+// - Graph pseudo-columns get table-level lineage (conservative direction:+// $from_id/$to_id encode referenced node-row identity).+// - $ROWGUID stays unresolved (fail-closed) until the catalog carries a+// rowguid flag.+func TestOmniQuerySpanPseudoColumns(t *testing.T) {+ identSrc := base.ColumnResource{Database: "db", Schema: "dbo", Table: "ident_t", Column: "id"}++ t.Run("identity_bare", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM ident_t")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ // Engine-verified: the result-set header for SELECT $IDENTITY is the+ // real column name, not "$IDENTITY".+ require.Equal(t, "id", span.Results[0].Name)+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)+ })++ t.Run("identitycol_keyword", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT IDENTITYCOL FROM ident_t")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)+ })++ t.Run("identity_qualified_alias", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT x.$IDENTITY FROM ident_t AS x")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)+ })++ t.Run("identity_qualified_disambiguates", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(),+ "SELECT ident_t.$IDENTITY FROM ident_t, ident_t2")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)+ })++ t.Run("identity_in_predicate", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT payload FROM ident_t WHERE $IDENTITY = 5")+ require.NoError(t, err)+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.PredicateColumns)+ })++ t.Run("identity_ambiguous_errors", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM ident_t, ident_t2")+ require.ErrorContains(t, err, "ambiguous")+ })++ t.Run("identity_no_identity_column_errors", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM t")+ require.ErrorContains(t, err, "no identity column")+ })++ // Graph result Name stays the written form ("$node_id"): the engine's+ // actual header is the mangled internal column ($node_id_<hex>), which the+ // synced catalog cannot reproduce.+ t.Run("graph_table_level_lineage", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT $node_id FROM t")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Schema: "dbo", Table: "t", Column: "a"}: true,+ {Database: "db", Schema: "dbo", Table: "t", Column: "b"}: true,+ {Database: "db", Schema: "dbo", Table: "t", Column: "c"}: true,+ }, span.Results[0].SourceColumns)+ })++ t.Run("graph_qualified_two_tables", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT t2.$from_id FROM t, t2")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Schema: "dbo", Table: "t2", Column: "a"}: true,+ {Database: "db", Schema: "dbo", Table: "t2", Column: "b"}: true,+ }, span.Results[0].SourceColumns)+ })++ t.Run("graph_bare_ambiguous_errors", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $node_id FROM t, t2")+ require.ErrorContains(t, err, "ambiguous")+ })++ t.Run("graph_empty_table_fails_closed", func(t *testing.T) {+ // Attribute-less edge tables have no catalog columns; empty lineage+ // would mean NoneMasker downstream, so this must error.+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $edge_id FROM bare_edge")+ require.ErrorContains(t, err, "no columns in the catalog")+ })++ // Delimited identifiers are never pseudo-columns: [IDENTITYCOL] and+ // [$node_id] reference real columns with those names.+ t.Run("delimited_identifiers_are_real_columns", func(t *testing.T) {+ for _, sql := range []string{+ "SELECT [$node_id] FROM weird",+ "SELECT weird.[$node_id] FROM weird",+ } {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), sql)+ require.NoError(t, err, sql)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Schema: "dbo", Table: "weird", Column: "$node_id"}: true,+ }, span.Results[0].SourceColumns, sql)+ }+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT [IDENTITYCOL] FROM weird")+ require.NoError(t, err)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Schema: "dbo", Table: "weird", Column: "IDENTITYCOL"}: true,+ }, span.Results[0].SourceColumns)+ })++ // A delimited TABLE with a bare column segment stays a pseudo-column:+ // [weird].$node_id is the graph reference, not the real $node_id column.+ t.Run("delimited_table_bare_column_stays_pseudo", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT [weird].$node_id FROM weird")+ require.NoError(t, err)+ require.Len(t, span.Results, 1)+ require.Equal(t, base.SourceColumnSet{+ {Database: "db", Schema: "dbo", Table: "weird", Column: "$node_id"}: true,+ {Database: "db", Schema: "dbo", Table: "weird", Column: "IDENTITYCOL"}: true,+ }, span.Results[0].SourceColumns)+ })++ t.Run("rowguid_stays_fail_closed", func(t *testing.T) {+ q := newOmniTestExtractor(t, "db")+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $ROWGUID FROM t")+ require.Error(t, err)+ })+}