Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: bound SQL completion clause parsing to the caret's statement (BYT-9886) (#20918)
h3n4l56e141f9e1c1
diff --git a/.sonarcloud.properties b/.sonarcloud.propertiesindex 33738beb20..88e5908421 100644--- a/.sonarcloud.properties+++ b/.sonarcloud.properties@@ -70,7 +70,10 @@ sonar.issue.ignore.multicriteria.fp_dump_sql.resourceKey=backend/plugin/db/*/dumsonar.issue.ignore.multicriteria.fp_role_sql.ruleKey=go:S2077sonar.issue.ignore.multicriteria.fp_role_sql.resourceKey=backend/plugin/db/*/role.go-# Exclusions for copy-paste detection+# Exclusions for copy-paste detection.+# backend/plugin/db/** and the per-engine parser completers are intentionally+# parallel per-engine implementations (one folder per database engine), so+# cross-engine similarity is by design, not copy-paste debt.sonar.cpd.exclusions=\action/**/*_test.go,\backend/**/*_test.go,\@@ -79,6 +82,7 @@ sonar.cpd.exclusions=\backend/generated-go/**,\backend/api/mcp/gen/**,\backend/plugin/db/**,\+ backend/plugin/parser/*/completion.go,\frontend/src/**/*.test.ts,\frontend/src/**/*.test.tsx,\frontend/scripts/**/*.test.mjs,\diff --git a/backend/plugin/parser/base/statement.go b/backend/plugin/parser/base/statement.goindex 2d085b6641..f7670d6c96 100644--- a/backend/plugin/parser/base/statement.go+++ b/backend/plugin/parser/base/statement.go@@ -32,6 +32,23 @@ func (s *Statement) BaseLine() int {return int(s.Start.Line) - 1}+// TruncateAtSemicolonToken returns sql[start:end], where end is the Loc of+// the first token at or after start whose type is ';' (the end of the+// statement containing start), or len(sql) when there is none. Scanning the+// token stream rather than raw bytes means a ';' inside a string literal,+// comment, or quoted identifier never counts as a statement boundary.+// Completers use this to bound clause fragments to the current statement so+// fragment re-parsing cannot scale with the size of the whole sheet+// (BYT-9886). loc and typ adapt the engine-specific token type.+func TruncateAtSemicolonToken[T any](sql string, start int, tokens []T, loc func(T) int, typ func(T) int) string {+ for _, tok := range tokens {+ if loc(tok) >= start && typ(tok) == ';' {+ return sql[start:loc(tok)]+ }+ }+ return sql[start:]+}+// NewStatementFromRange builds a Statement from a split segment range.func NewStatementFromRange(sql string, mapper *ByteOffsetPositionMapper, byteStart, byteEnd int, empty bool) Statement {if byteEnd < len(sql) && sql[byteEnd] == ';' {diff --git a/backend/plugin/parser/mysql/completion.go b/backend/plugin/parser/mysql/completion.goindex f5b6925bbc..c49ebedc78 100644--- a/backend/plugin/parser/mysql/completion.go+++ b/backend/plugin/parser/mysql/completion.go@@ -606,7 +606,7 @@ func (c *Completer) extractCTETables(pos int) []*base.VirtualTableReference {if pos >= len(c.sql) {return nil}- followingText := c.sql[pos:]+ followingText := c.statementBoundedTextFromByte(pos)if len(followingText) == 0 {return nil}@@ -850,6 +850,22 @@ func (c *Completer) takeReferencesSnapshot() {}}+// statementBoundedText returns the text of c.sql starting at token i and+// truncated at the next ';' token. Fragment re-parsing (parseTableReferences,+// extractCTETables) tries progressively shorter token prefixes, so feeding it+// text past the end of the current statement makes its cost quadratic in the+// size of the whole sheet instead of the current statement (BYT-9886).+func (c *Completer) statementBoundedText(i int) string {+ return c.statementBoundedTextFromByte(c.tokens[i].Loc)+}++// statementBoundedTextFromByte is statementBoundedText for a byte offset.+func (c *Completer) statementBoundedTextFromByte(pos int) string {+ return base.TruncateAtSemicolonToken(c.sql, pos, c.tokens,+ func(t mysqlparser.Token) int { return t.Loc },+ func(t mysqlparser.Token) int { return t.Type })+}+func (c *Completer) collectRemainingTableReferences() {level := 0for i := c.caretTokenIndex; i < len(c.tokens); i++ {@@ -860,9 +876,13 @@ func (c *Completer) collectRemainingTableReferences() {if level > 0 {level--}+ case ';':+ // End of the caret's statement: later statements' tables are not+ // in scope for the caret and must not leak into its completion.+ returncase mysqlparser.FROM:if level == 0 {- c.parseTableReferences(c.sql[c.tokens[i].Loc:])+ c.parseTableReferences(c.statementBoundedText(i))}default:}@@ -883,9 +903,9 @@ func (c *Completer) collectLeadingTableReferences(caretIndex int) {level--c.referencesStack = c.referencesStack[1:]case mysqlparser.FROM:- c.parseTableReferences(c.sql[c.tokens[i].Loc:])+ c.parseTableReferences(c.statementBoundedText(i))case mysqlparser.INSERT, mysqlparser.INTO:- c.parseInsertTableReferences(c.sql[c.tokens[i].Loc:])+ c.parseInsertTableReferences(c.statementBoundedText(i))default:// ALTER TABLE / TRUNCATE TABLE / DROP TABLE / UPDATE / DELETE FROM// — bind the target table for column completion in DDL/DML contexts.diff --git a/backend/plugin/parser/mysql/completion_test.go b/backend/plugin/parser/mysql/completion_test.goindex c99a3426c3..782246e413 100644--- a/backend/plugin/parser/mysql/completion_test.go+++ b/backend/plugin/parser/mysql/completion_test.go@@ -2,10 +2,13 @@ package mysqlimport ("context"+ "fmt""io""os""slices"+ "strings""testing"+ "time""github.com/stretchr/testify/require""gopkg.in/yaml.v3"@@ -91,6 +94,44 @@ func TestCompletion(t *testing.T) {}}+// Completion cost must be bounded by the caret's statement, not the whole+// sheet: a broken statement anywhere after the caret must not make the+// FROM-clause re-parse loop shrink through the entire trailing document+// (BYT-9886).+func TestCompletionWithBrokenTrailingStatementScalesLinearly(t *testing.T) {+ a := require.New(t)++ // The caret completes through the JOIN alias a2: resolving a2 to t2's+ // columns requires parseTableReferences to actually extract the FROM+ // clause, so an over-truncated (empty) fragment cannot pass this test.+ var sheet strings.Builder+ sheet.WriteString("SELECT a2. FROM t1 JOIN t2 a2 ON t1.c1 = a2.c1;\n")+ sheet.WriteString("SELEC broken FROM oops;\n")+ for i := range 800 {+ fmt.Fprintf(&sheet, "SELECT col_a, col_b, col_c FROM table_%04d WHERE col_a = %d AND col_b LIKE 'pattern%%' ORDER BY col_c LIMIT 100;\n", i, i)+ }++ started := time.Now()+ result, err := base.Completion(context.Background(), storepb.Engine_MYSQL, base.CompletionContext{+ Scene: base.SceneTypeAll,+ DefaultDatabase: "db",+ Metadata: getMetadataForTest,+ ListDatabaseNames: listDatabaseNamesForTest,+ }, sheet.String(), 1, 10 /* caret right after "SELECT a2." */)+ elapsed := time.Since(started)++ a.NoError(err)+ var texts []string+ for _, candidate := range result {+ if candidate.Type == base.CandidateTypeColumn {+ texts = append(texts, candidate.Text)+ }+ }+ a.Contains(texts, "c1")+ a.Contains(texts, "c2")+ a.Less(elapsed, 2*time.Second)+}+func listDatabaseNamesForTest(_ context.Context, _ string) ([]string, error) {return []string{"db"}, nil}diff --git a/backend/plugin/parser/pg/completion.go b/backend/plugin/parser/pg/completion.goindex 10cea9abd2..37108fa466 100644--- a/backend/plugin/parser/pg/completion.go+++ b/backend/plugin/parser/pg/completion.go@@ -767,7 +767,7 @@ func (c *Completer) extractCTETables(pos int) []*base.VirtualTableReference {if pos >= len(c.sql) {return nil}- followingText := c.sql[pos:]+ followingText := c.statementBoundedTextFromByte(pos)if len(followingText) == 0 {return nil}@@ -1222,6 +1222,20 @@ func (c *Completer) convertCompletionSubqueryReference(reference pgparser.RangeRreturn virtualReference}+// statementBoundedText returns the text of c.sql starting at token i and+// truncated at the next ';' token, bounding fragment re-parsing to the+// current statement (BYT-9886).+func (c *Completer) statementBoundedText(i int) string {+ return c.statementBoundedTextFromByte(c.tokens[i].Loc)+}++// statementBoundedTextFromByte is statementBoundedText for a byte offset.+func (c *Completer) statementBoundedTextFromByte(pos int) string {+ return base.TruncateAtSemicolonToken(c.sql, pos, c.tokens,+ func(t pgparser.Token) int { return t.Loc },+ func(t pgparser.Token) int { return t.Type })+}+func (c *Completer) collectRemainingTableReferences() {level := 0for i := c.caretTokenIndex; i < len(c.tokens); i++ {@@ -1232,9 +1246,13 @@ func (c *Completer) collectRemainingTableReferences() {if level > 0 {level--}+ case ';':+ // End of the caret's statement: later statements' tables are not+ // in scope for the caret and must not leak into its completion.+ returncase pgparser.FROM:if level == 0 {- c.parseTableReferences(c.sql[c.tokens[i].Loc:])+ c.parseTableReferences(c.statementBoundedText(i))}default:}@@ -1255,7 +1273,7 @@ func (c *Completer) collectLeadingTableReferences(caretIndex int) {level--c.referencesStack = c.referencesStack[1:]case pgparser.FROM:- c.parseTableReferences(c.sql[c.tokens[i].Loc:])+ c.parseTableReferences(c.statementBoundedText(i))default:}}diff --git a/backend/plugin/parser/pg/completion_test.go b/backend/plugin/parser/pg/completion_test.goindex 4ab2cbc3f0..92868bee6c 100644--- a/backend/plugin/parser/pg/completion_test.go+++ b/backend/plugin/parser/pg/completion_test.go@@ -2,10 +2,13 @@ package pgimport ("context"+ "fmt""io""os""slices"+ "strings""testing"+ "time""github.com/stretchr/testify/require""gopkg.in/yaml.v3"@@ -95,6 +98,44 @@ func listDatbaseNamesForTest(_ context.Context, _ string) ([]string, error) {return []string{"db"}, nil}+// Completion cost must be bounded by the caret's statement, not the whole+// sheet: a broken statement anywhere after the caret must not make the+// FROM-clause re-parse loop shrink through the entire trailing document+// (BYT-9886).+func TestCompletionWithBrokenTrailingStatementScalesLinearly(t *testing.T) {+ a := require.New(t)++ // The caret is unqualified, so column candidates can only come from the+ // tables parseTableReferences extracts out of the FROM clause; an+ // over-truncated (empty) fragment cannot pass this test.+ var sheet strings.Builder+ sheet.WriteString("SELECT FROM t1 JOIN t2 a2 ON t1.c1 = a2.c1;\n")+ sheet.WriteString("SELEC broken FROM oops;\n")+ for i := range 2000 {+ fmt.Fprintf(&sheet, "SELECT col_a, col_b, col_c FROM table_%04d WHERE col_a = %d AND col_b LIKE 'pattern%%' ORDER BY col_c LIMIT 100;\n", i, i)+ }++ started := time.Now()+ result, err := base.Completion(context.Background(), storepb.Engine_POSTGRES, base.CompletionContext{+ Scene: base.SceneTypeAll,+ DefaultDatabase: "db",+ Metadata: getMetadataForTest,+ ListDatabaseNames: listDatbaseNamesForTest,+ }, sheet.String(), 1, 7 /* caret in the select list, right after "SELECT " */)+ elapsed := time.Since(started)++ a.NoError(err)+ var texts []string+ for _, candidate := range result {+ if candidate.Type == base.CandidateTypeColumn {+ texts = append(texts, candidate.Text)+ }+ }+ a.Contains(texts, "c1")+ a.Contains(texts, "c2")+ a.Less(elapsed, 2*time.Second)+}+func getMetadataForTest(_ context.Context, _, databaseName string) (string, *model.DatabaseMetadata, error) {if databaseName != "db" {return "", nil, nil