Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(tsql): resolve pseudo-columns in query span; suggest $action in completion (#20754)

rebelice07b539afe8b7
diff --git a/backend/plugin/parser/tsql/completion.go b/backend/plugin/parser/tsql/completion.go
index f6551b6968..a0f7591daf 100644
--- a/backend/plugin/parser/tsql/completion.go
+++ b/backend/plugin/parser/tsql/completion.go
@@ -777,6 +777,13 @@ func (c *Completer) convertCandidates(candidates *mssqlparser.CandidateSet) ([]b
for _, context := range completionContexts {
c.insertObjectCandidates(context, databaseEntries, schemaEntries, tableEntries, viewEntries, sequenceEntries)
}
+ case "pseudocolumn_action":
+ // OUTPUT clause position: suggest the $action pseudo-column
+ // (MERGE OUTPUT returns 'INSERT'/'UPDATE'/'DELETE' per row).
+ columnEntries.Insert(base.Candidate{
+ Type: base.CandidateTypeColumn,
+ Text: "$action",
+ })
case "columnref":
completionContexts := c.determineColumnNameContext()
for _, context := range completionContexts {
diff --git a/backend/plugin/parser/tsql/completion_pseudo_column_test.go b/backend/plugin/parser/tsql/completion_pseudo_column_test.go
new file mode 100644
index 0000000000..a628720abd
--- /dev/null
+++ b/backend/plugin/parser/tsql/completion_pseudo_column_test.go
@@ -0,0 +1,30 @@
+package tsql
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// TestCompletionPseudoColumnAction pins that the OUTPUT-clause position
+// surfaces the $action pseudo-column: omni emits the pseudocolumn_action rule
+// there, which previously fell through Bytebase's rule switch silently.
+func TestCompletionPseudoColumnAction(t *testing.T) {
+ statement, caretLine, caretPosition := getCaretPosition(
+ "MERGE INTO t AS d USING s ON d.k = s.k WHEN MATCHED THEN UPDATE SET d.v = s.v OUTPUT |")
+ getter, lister := buildMockDatabaseMetadataGetterLister()
+ results, err := Completion(context.Background(), base.CompletionContext{
+ Scene: base.SceneTypeAll,
+ DefaultDatabase: "Company",
+ Metadata: getter,
+ ListDatabaseNames: lister,
+ }, statement, caretLine, caretPosition)
+ require.NoError(t, err)
+ require.Contains(t, results, base.Candidate{
+ Type: base.CandidateTypeColumn,
+ Text: "$action",
+ })
+}
diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni.go b/backend/plugin/parser/tsql/query_span_extractor_omni.go
index af6d0d0c6b..0f75539b56 100644
--- a/backend/plugin/parser/tsql/query_span_extractor_omni.go
+++ b/backend/plugin/parser/tsql/query_span_extractor_omni.go
@@ -192,6 +192,13 @@ func (q *omniQuerySpanExtractor) collectPredicatesInScope(expr ast.Node) error {
}
switch v := expr.(type) {
case *ast.ColumnRef:
+ if r, handled, err := q.resolvePseudoColumnRef(v); handled {
+ if err != nil {
+ return errors.Wrapf(err, "failed to resolve predicate column %q", v.Column)
+ }
+ q.predicateColumns, _ = base.MergeSourceColumnSet(q.predicateColumns, r.SourceColumns)
+ return nil
+ }
r, err := q.tsqlIsFieldSensitive(v.Database, v.Schema, v.Table, v.Column)
if err != nil {
return errors.Wrapf(err, "failed to resolve predicate column %q", v.Column)
@@ -1001,6 +1008,12 @@ func (q *omniQuerySpanExtractor) resolveExpressionNode(n ast.Node) (base.QuerySp
}
switch v := n.(type) {
case *ast.ColumnRef:
+ if r, handled, err := q.resolvePseudoColumnRef(v); handled {
+ if err != nil {
+ return base.QuerySpanResult{}, errors.Wrapf(err, "failed to resolve column %q", v.Column)
+ }
+ return r, nil
+ }
r, err := q.tsqlIsFieldSensitive(v.Database, v.Schema, v.Table, v.Column)
if err != nil {
return base.QuerySpanResult{}, errors.Wrapf(err, "failed to resolve column %q", v.Column)
diff --git a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
index 922d86c6a8..be55586fd1 100644
--- a/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
+++ b/backend/plugin/parser/tsql/query_span_extractor_omni_test.go
@@ -25,6 +25,10 @@ var omniTestMetadata = []*storepb.DatabaseSchemaMetadata{
{Name: "t", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}, {Name: "c"}}},
{Name: "t1", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}, {Name: "c"}}},
{Name: "t2", Columns: []*storepb.ColumnMetadata{{Name: "a"}, {Name: "b"}}},
+ {Name: "ident_t", Columns: []*storepb.ColumnMetadata{{Name: "id", IsIdentity: true}, {Name: "payload"}}},
+ {Name: "ident_t2", Columns: []*storepb.ColumnMetadata{{Name: "seq", IsIdentity: true}, {Name: "v"}}},
+ {Name: "bare_edge"},
+ {Name: "weird", Columns: []*storepb.ColumnMetadata{{Name: "$node_id"}, {Name: "IDENTITYCOL"}}},
},
Views: []*storepb.ViewMetadata{
{Name: "vw", Definition: "CREATE VIEW [dbo].[vw] AS SELECT a, b FROM t"},
diff --git a/backend/plugin/parser/tsql/query_span_pseudo_column_omni.go b/backend/plugin/parser/tsql/query_span_pseudo_column_omni.go
new file mode 100644
index 0000000000..97332c8d7a
--- /dev/null
+++ b/backend/plugin/parser/tsql/query_span_pseudo_column_omni.go
@@ -0,0 +1,211 @@
+package tsql
+
+import (
+ "strings"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+
+ "github.com/bytebase/omni/mssql/ast"
+)
+
+// pseudoColumnKind classifies T-SQL pseudo-column references the omni parser
+// accepts as ColumnRef nodes.
+type pseudoColumnKind int
+
+const (
+ pseudoColumnNone pseudoColumnKind = iota
+ // pseudoColumnIdentity is $IDENTITY / IDENTITYCOL: the engine resolves it
+ // to the table's single IDENTITY-property column at bind time.
+ pseudoColumnIdentity
+ // pseudoColumnGraph is $node_id / $edge_id / $from_id / $to_id on graph
+ // tables. The backing engine columns are auto-generated and not present in
+ // the synced catalog.
+ pseudoColumnGraph
+)
+
+func classifyPseudoColumn(name string) pseudoColumnKind {
+ switch strings.ToLower(name) {
+ case "$identity", "identitycol":
+ return pseudoColumnIdentity
+ case "$node_id", "$edge_id", "$from_id", "$to_id":
+ return pseudoColumnGraph
+ default:
+ // $ROWGUID / ROWGUIDCOL deliberately unhandled: the synced catalog has
+ // no rowguid flag yet, so they keep today's fail-closed resolution
+ // error until metadata support lands.
+ return pseudoColumnNone
+ }
+}
+
+// resolvePseudoColumnRef maps a pseudo-column reference to real column
+// lineage. handled is false when v is not a (supported) pseudo-column and the
+// ordinary resolution path should run.
+//
+// - $IDENTITY / IDENTITYCOL map precisely to the in-scope table's column
+// with ColumnMetadata.IsIdentity, so masking rules on the identity column
+// apply to the pseudo-column reference too.
+// - Graph pseudo-columns get table-level lineage (the union of the table's
+// columns): $from_id/$to_id encode the identity of referenced node rows,
+// so empty lineage would fail open in scenarios where relationships
+// themselves are sensitive; over-masking is the safer direction.
+//
+// Both kinds require the reference to bind to exactly one candidate table in
+// scope — the engine reports ambiguity errors itself, so mirroring that is
+// fail-closed.
+func (q *omniQuerySpanExtractor) resolvePseudoColumnRef(v *ast.ColumnRef) (base.QuerySpanResult, bool, error) {
+ kind := classifyPseudoColumn(v.Column)
+ if kind == pseudoColumnNone {
+ return base.QuerySpanResult{}, false, nil
+ }
+ // Delimited identifiers are never pseudo-columns in T-SQL: [IDENTITYCOL]
+ // and [$node_id] reference real columns with those names. omni strips the
+ // delimiters from ColumnRef.Column, so check the source text — a
+ // delimited column segment makes the ref end with ']' or '"', which a
+ // bare pseudo-column or keyword never does.
+ if q.columnSegmentIsDelimited(v) {
+ return base.QuerySpanResult{}, false, nil
+ }
+
+ sources := q.pseudoColumnCandidateSources(v)
+ switch kind {
+ case pseudoColumnIdentity:
+ r, err := q.resolveIdentityPseudoColumn(v, sources)
+ return r, true, err
+ case pseudoColumnGraph:
+ r, err := q.resolveGraphPseudoColumn(v, sources)
+ return r, true, err
+ default:
+ return base.QuerySpanResult{}, false, nil
+ }
+}
+
+// columnSegmentIsDelimited reports whether the reference's column segment is
+// bracket- or quote-delimited in the original source.
+func (q *omniQuerySpanExtractor) columnSegmentIsDelimited(v *ast.ColumnRef) bool {
+ end := v.Loc.End
+ if end <= 0 || end > len(q.source) {
+ return false
+ }
+ last := q.source[end-1]
+ return last == ']' || last == '"'
+}
+
+// pseudoColumnCandidateSources returns the in-scope table sources the
+// reference may bind to, filtered by the optional table qualifier
+// (t.$IDENTITY binds within t only). Inner scope is searched before outer,
+// matching tsqlIsFieldSensitive.
+func (q *omniQuerySpanExtractor) pseudoColumnCandidateSources(v *ast.ColumnRef) []base.TableSource {
+ matches := func(ts base.TableSource) bool {
+ if v.Table != "" && !q.isIdentifierEqual(v.Table, ts.GetTableName()) {
+ return false
+ }
+ if v.Schema != "" && !q.isIdentifierEqual(v.Schema, ts.GetSchemaName()) {
+ return false
+ }
+ if v.Database != "" && !q.isIdentifierEqual(v.Database, ts.GetDatabaseName()) {
+ return false
+ }
+ return true
+ }
+ var result []base.TableSource
+ for _, ts := range q.tableSourcesFrom {
+ if matches(ts) {
+ result = append(result, ts)
+ }
+ }
+ if len(result) > 0 {
+ return result
+ }
+ for i := len(q.outerTableSources) - 1; i >= 0; i-- {
+ if matches(q.outerTableSources[i]) {
+ result = append(result, q.outerTableSources[i])
+ }
+ }
+ return result
+}
+
+func (q *omniQuerySpanExtractor) resolveIdentityPseudoColumn(v *ast.ColumnRef, sources []base.TableSource) (base.QuerySpanResult, error) {
+ type match struct {
+ column base.QuerySpanResult
+ }
+ var matches []match
+ for _, ts := range sources {
+ for _, column := range ts.GetQuerySpanResult() {
+ if q.isIdentityColumnResult(column) {
+ matches = append(matches, match{column: column})
+ break // a table has at most one identity column
+ }
+ }
+ }
+ switch len(matches) {
+ case 0:
+ return base.QuerySpanResult{}, errors.Errorf("no identity column found in scope for %q", v.Column)
+ case 1:
+ r := matches[0].column
+ r.IsPlainField = true
+ return r, nil
+ default:
+ return base.QuerySpanResult{}, errors.Errorf("ambiguous %q: multiple tables in scope have an identity column", v.Column)
+ }
+}
+
+// isIdentityColumnResult reports whether a table-source column is backed by
+// exactly one physical column whose metadata has IsIdentity. The lookup goes
+// through the column's source resource rather than the table source itself so
+// aliased tables (wrapped as PseudoTable) resolve too.
+func (q *omniQuerySpanExtractor) isIdentityColumnResult(column base.QuerySpanResult) bool {
+ if !column.IsPlainField || len(column.SourceColumns) != 1 {
+ return false
+ }
+ for resource := range column.SourceColumns {
+ if resource.Database == "" || resource.Table == "" || resource.Column == "" {
+ return false
+ }
+ _, databaseMeta, err := q.gCtx.GetDatabaseMetadataFunc(q.ctx, q.gCtx.InstanceID, resource.Database)
+ if err != nil || databaseMeta == nil {
+ return false
+ }
+ schemaMeta := databaseMeta.GetSchemaMetadata(resource.Schema)
+ if schemaMeta == nil {
+ return false
+ }
+ tableMeta := schemaMeta.GetTable(resource.Table)
+ if tableMeta == nil {
+ return false
+ }
+ for _, columnMeta := range tableMeta.GetProto().GetColumns() {
+ if q.isIdentifierEqual(columnMeta.Name, resource.Column) {
+ return columnMeta.IsIdentity
+ }
+ }
+ }
+ return false
+}
+
+func (*omniQuerySpanExtractor) resolveGraphPseudoColumn(v *ast.ColumnRef, sources []base.TableSource) (base.QuerySpanResult, error) {
+ if len(sources) == 0 {
+ return base.QuerySpanResult{}, errors.Errorf("no table in scope for graph pseudo-column %q", v.Column)
+ }
+ if len(sources) > 1 && v.Table == "" {
+ return base.QuerySpanResult{}, errors.Errorf("ambiguous graph pseudo-column %q: qualify it with the table name", v.Column)
+ }
+ sourceColumns := make(base.SourceColumnSet)
+ for _, column := range sources[0].GetQuerySpanResult() {
+ sourceColumns, _ = base.MergeSourceColumnSet(sourceColumns, column.SourceColumns)
+ }
+ // Attribute-less graph tables (CREATE TABLE ... AS EDGE with no
+ // user-defined columns) contribute no catalog columns, so the union is
+ // empty — and empty SourceColumns downstream means NoneMasker, the exact
+ // fail-open this table-level lineage is meant to prevent. Fail closed: a
+ // table-only resource would not match column-level masking rules either.
+ if len(sourceColumns) == 0 {
+ return base.QuerySpanResult{}, errors.Errorf("cannot derive lineage for graph pseudo-column %q: table has no columns in the catalog", v.Column)
+ }
+ return base.QuerySpanResult{
+ Name: v.Column,
+ SourceColumns: sourceColumns,
+ IsPlainField: false,
+ }, nil
+}
diff --git a/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.go b/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.go
new file mode 100644
index 0000000000..de52d99f41
--- /dev/null
+++ b/backend/plugin/parser/tsql/query_span_pseudo_column_omni_test.go
@@ -0,0 +1,160 @@
+package tsql
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// TestOmniQuerySpanPseudoColumns covers the downstream mapping for T-SQL
+// pseudo-columns the parser accepts since the $-token omni bump:
+// - $IDENTITY / IDENTITYCOL map precisely to the in-scope table's
+// ColumnMetadata.IsIdentity column, so masking rules on the identity
+// column follow the pseudo-column reference.
+// - Graph pseudo-columns get table-level lineage (conservative direction:
+// $from_id/$to_id encode referenced node-row identity).
+// - $ROWGUID stays unresolved (fail-closed) until the catalog carries a
+// rowguid flag.
+func TestOmniQuerySpanPseudoColumns(t *testing.T) {
+ identSrc := base.ColumnResource{Database: "db", Schema: "dbo", Table: "ident_t", Column: "id"}
+
+ t.Run("identity_bare", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM ident_t")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ // Engine-verified: the result-set header for SELECT $IDENTITY is the
+ // real column name, not "$IDENTITY".
+ require.Equal(t, "id", span.Results[0].Name)
+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)
+ })
+
+ t.Run("identitycol_keyword", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT IDENTITYCOL FROM ident_t")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)
+ })
+
+ t.Run("identity_qualified_alias", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT x.$IDENTITY FROM ident_t AS x")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)
+ })
+
+ t.Run("identity_qualified_disambiguates", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(),
+ "SELECT ident_t.$IDENTITY FROM ident_t, ident_t2")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.Results[0].SourceColumns)
+ })
+
+ t.Run("identity_in_predicate", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT payload FROM ident_t WHERE $IDENTITY = 5")
+ require.NoError(t, err)
+ require.Equal(t, base.SourceColumnSet{identSrc: true}, span.PredicateColumns)
+ })
+
+ t.Run("identity_ambiguous_errors", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM ident_t, ident_t2")
+ require.ErrorContains(t, err, "ambiguous")
+ })
+
+ t.Run("identity_no_identity_column_errors", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $IDENTITY FROM t")
+ require.ErrorContains(t, err, "no identity column")
+ })
+
+ // Graph result Name stays the written form ("$node_id"): the engine's
+ // actual header is the mangled internal column ($node_id_<hex>), which the
+ // synced catalog cannot reproduce.
+ t.Run("graph_table_level_lineage", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT $node_id FROM t")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{
+ {Database: "db", Schema: "dbo", Table: "t", Column: "a"}: true,
+ {Database: "db", Schema: "dbo", Table: "t", Column: "b"}: true,
+ {Database: "db", Schema: "dbo", Table: "t", Column: "c"}: true,
+ }, span.Results[0].SourceColumns)
+ })
+
+ t.Run("graph_qualified_two_tables", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT t2.$from_id FROM t, t2")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{
+ {Database: "db", Schema: "dbo", Table: "t2", Column: "a"}: true,
+ {Database: "db", Schema: "dbo", Table: "t2", Column: "b"}: true,
+ }, span.Results[0].SourceColumns)
+ })
+
+ t.Run("graph_bare_ambiguous_errors", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $node_id FROM t, t2")
+ require.ErrorContains(t, err, "ambiguous")
+ })
+
+ t.Run("graph_empty_table_fails_closed", func(t *testing.T) {
+ // Attribute-less edge tables have no catalog columns; empty lineage
+ // would mean NoneMasker downstream, so this must error.
+ q := newOmniTestExtractor(t, "db")
+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $edge_id FROM bare_edge")
+ require.ErrorContains(t, err, "no columns in the catalog")
+ })
+
+ // Delimited identifiers are never pseudo-columns: [IDENTITYCOL] and
+ // [$node_id] reference real columns with those names.
+ t.Run("delimited_identifiers_are_real_columns", func(t *testing.T) {
+ for _, sql := range []string{
+ "SELECT [$node_id] FROM weird",
+ "SELECT weird.[$node_id] FROM weird",
+ } {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), sql)
+ require.NoError(t, err, sql)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{
+ {Database: "db", Schema: "dbo", Table: "weird", Column: "$node_id"}: true,
+ }, span.Results[0].SourceColumns, sql)
+ }
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT [IDENTITYCOL] FROM weird")
+ require.NoError(t, err)
+ require.Equal(t, base.SourceColumnSet{
+ {Database: "db", Schema: "dbo", Table: "weird", Column: "IDENTITYCOL"}: true,
+ }, span.Results[0].SourceColumns)
+ })
+
+ // A delimited TABLE with a bare column segment stays a pseudo-column:
+ // [weird].$node_id is the graph reference, not the real $node_id column.
+ t.Run("delimited_table_bare_column_stays_pseudo", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ span, err := q.getOmniQuerySpan(context.Background(), "SELECT [weird].$node_id FROM weird")
+ require.NoError(t, err)
+ require.Len(t, span.Results, 1)
+ require.Equal(t, base.SourceColumnSet{
+ {Database: "db", Schema: "dbo", Table: "weird", Column: "$node_id"}: true,
+ {Database: "db", Schema: "dbo", Table: "weird", Column: "IDENTITYCOL"}: true,
+ }, span.Results[0].SourceColumns)
+ })
+
+ t.Run("rowguid_stays_fail_closed", func(t *testing.T) {
+ q := newOmniTestExtractor(t, "db")
+ _, err := q.getOmniQuerySpan(context.Background(), "SELECT $ROWGUID FROM t")
+ require.Error(t, err)
+ })
+}