Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(doris,starrocks): gate INTO OUTFILE and EXPLAIN ANALYZE off the AST (#21735)
Danny Xue26dc2d3bb61
diff --git a/backend/api/v1/mcp_sql_clamp_test.go b/backend/api/v1/mcp_sql_clamp_test.goindex 36e428630d..a1b4479800 100644--- a/backend/api/v1/mcp_sql_clamp_test.go+++ b/backend/api/v1/mcp_sql_clamp_test.go@@ -44,7 +44,7 @@ import (// unclassifiedEngineWrites are the engines with no registered read-only// classifier, each with a statement that writes on it. Every one fails closed-// under the clamp rather than gaining a validator in this change.+// under the clamp rather than gaining a validator.//// One map rather than a list beside a lookup table: two copies of this set// could disagree about which engines the test is even about.@@ -269,15 +269,18 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {reason: "the statement is not a read",},{- // Doris nests block comments; the shared stripper stops at the- // first "*/", so stripping first would leave text between INTO- // and OUTFILE and lose a keyword-pair match. The guard reads the- // raw statement for that reason.+ // Doris nests block comments and omni's Doris lexer does not: it+ // ends one at the first "*/", so the tail leaks back into the+ // token stream. What leaks is not a clause, so the statement does+ // not parse and is refused as a syntax error rather than reaching+ // the export gate. Doris, which nests, reads the same text as an+ // export, so the two disagree about what this statement is and it+ // is refused either way (BOT-93).name: "a nested comment cannot hide a Doris export",engine: storepb.Engine_DORIS,statement: "SELECT * FROM employee\nINTO /* outer /* inner */ tail */ OUTFILE \"s3://bucket/export/\"\nFORMAT AS PARQUET",refused: true,- reason: "the statement is not a read",+ reason: "the statement could not be parsed",},{name: "nor can a line comment",@@ -287,11 +290,16 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {reason: "the statement is not a read",},{+ // An unterminated literal means there is no tree to classify, so+ // the clamp refuses it as the syntax error it is. It was refused+ // as "not a read" while a token scan stood in for the export+ // gate, since that scan treated any lex error as a possible+ // hidden keyword.name: "an unlexable Doris statement fails closed",engine: storepb.Engine_DORIS,statement: "SELECT * FROM t WHERE a = 'unterminated",refused: true,- reason: "the statement is not a read",+ reason: "the statement could not be parsed",},{// USE rebinds the connection's catalog and schema, so an@@ -349,16 +357,17 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {},{// Not "inside a comment" for a NESTED one: omni's Doris lexer ends- // a comment at the first "*/", so the tail leaks back into the- // token stream and the keyword is seen. Doris itself nests, so- // this query is a plain SELECT 1 there and Bytebase refuses it- // anyway (BOT-93). Pinned as the cost of failing closed on a- // disagreement rather than guessing which lexer is right.+ // a comment at the first "*/", so "OUTFILE */" leaks back into the+ // token stream, where it parses as nothing and the statement is+ // refused as a syntax error. Doris itself nests, so this query is+ // a plain SELECT 1 there (BOT-93). Pinned as the cost of failing+ // closed on a disagreement rather than guessing which lexer is+ // right.name: "a nested comment holding the keyword refuses, though Doris would not",engine: storepb.Engine_DORIS,statement: "SELECT 1 /* outer /* inner */ OUTFILE */",refused: true,- reason: "the statement is not a read",+ reason: "the statement could not be parsed",},{name: "a read on an engine with no splitter is still classified",diff --git a/backend/plugin/parser/doris/limit.go b/backend/plugin/parser/doris/limit.goindex 07935e365a..c256641d69 100644--- a/backend/plugin/parser/doris/limit.go+++ b/backend/plugin/parser/doris/limit.go@@ -42,14 +42,21 @@ func statementWithResultLimit(statement string, limit int, _ string) string {return stmt}-// The Doris AST cannot represent INTO OUTFILE. Keep this top-level clause outside-// the query while capping it, including when a set operation needs wrapping.+// splitOutfileClause cuts a top-level INTO OUTFILE clause off the end of the+// statement, so the clause stays outside the query while it is capped,+// including when a set operation needs wrapping.+//+// This reads tokens rather than the tree the clause now has (omni's+// doris/ast.IntoOutfileClause) because it has to answer for text that does+// not parse: a statement whose parse fails still reaches the caller's+// CTE-wrap fallback, which needs the clause held back from the wrap.func splitOutfileClause(statement string) (string, string) {tokens, errs := parser.Tokenize(statement)if len(errs) > 0 {return statement, ""}intoKind, _ := parser.KeywordToken("into")+ outfileKind, _ := parser.KeywordToken("outfile")depth := 0for i, token := range tokens {switch token.Kind {@@ -60,7 +67,7 @@ func splitOutfileClause(statement string) (string, string) {case ';':return statement, ""case intoKind:- if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileTokenKind {+ if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileKind {return statement[:token.Loc.Start], statement[token.Loc.Start:]}default:diff --git a/backend/plugin/parser/doris/query.go b/backend/plugin/parser/doris/query.goindex 0e35cfa64a..12b5836223 100644--- a/backend/plugin/parser/doris/query.go+++ b/backend/plugin/parser/doris/query.go@@ -2,7 +2,6 @@ package dorisimport ("github.com/bytebase/omni/doris/ast"- "github.com/bytebase/omni/doris/parser"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -20,6 +19,11 @@ func init() {// alone would mis-accept CTE-prefixed DML such as `WITH x AS (...) UPDATE ...`,// which Classify would tag as SELECT because `WITH` is its first token.//+// A query that carries INTO OUTFILE is refused: it writes its result set to+// S3 or HDFS, so the data leaves the database past masking and past the+// export gate. isReadOnlyAST reads the clause off the tree, so the words in a+// comment or a quoted identifier are not mistaken for one (BOT-92).+//// Syntax errors are surfaced up; that lets validateQueryRequest reject// malformed read-only SQL before execution.//@@ -28,14 +32,6 @@ func init() {// false once a statement needs executing rather than reading, which is how the// driver decides between QueryContext and ExecContext.func validateQuery(statement string) (bool, bool, error) {- // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.- // Matched on text because omni's Doris grammar has no INTO node: it parses- // to a bare SelectStmt identical to one without the clause (BOT-92).- // Replace with an AST check, as starrocks/query.go does, once it has one.- //- if mentionsFileExport(statement) {- return false, false, nil- }parsed, err := parseDorisSQL(statement)if err != nil {return false, false, err@@ -84,12 +80,17 @@ func needsExecute(node ast.Node) bool {// SQL after parseDorisSQL succeeds.func isReadOnlyAST(node ast.Node) bool {switch n := node.(type) {- case *ast.SelectStmt, *ast.SetOpStmt:- return true+ case *ast.SelectStmt:+ // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.+ return n.Into == nil+ case *ast.SetOpStmt:+ return n.Into == nilcase *ast.GroupedQuery:// A repeated trailing clause group or a parenthesized WITH group —// (SELECT 1 LIMIT 1) LIMIT 2 — wraps a query; classify by the query.- return isReadOnlyAST(n.Query)+ // The export gate applies to the group as well, since the clause+ // attaches to whichever node is the statement's outermost query.+ return n.Into == nil && isReadOnlyAST(n.Query)case *ast.ShowStmt:// Reject bare `SHOW` (Type is empty when the parser took the stub path// without seeing a recognised variant keyword).@@ -116,51 +117,25 @@ func isReadOnlyAST(node ast.Node) bool {// isExplainableInner reports whether `node` is a shape that EXPLAIN can// legitimately wrap (SELECT family or DML). DDL inside EXPLAIN is rejected:// Doris only supports EXPLAIN on query and DML statements.+//+// An export stays refused under EXPLAIN. The engine does not treat+// `EXPLAIN SELECT ... INTO OUTFILE ...` as a plain plan: it fails with the+// same "Please specify BROKER properties or check your local file path" as+// the bare export, where `EXPLAIN SELECT 1` returns a plan (container-+// verified). So the clause is not inert here, and the gate holds.func isExplainableInner(node ast.Node) bool {switch n := node.(type) {- case *ast.SelectStmt, *ast.SetOpStmt:- return true+ case *ast.SelectStmt:+ return n.Into == nil+ case *ast.SetOpStmt:+ return n.Into == nilcase *ast.GroupedQuery:// EXPLAIN (SELECT 1 LIMIT 1) LIMIT 2 — engine-verified accept;// classify by the wrapped query.- return isExplainableInner(n.Query)+ return n.Into == nil && isExplainableInner(n.Query)case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt,*ast.MergeStmt, *ast.TruncateTableStmt:return true}return false}--// mentionsFileExport reports whether the statement carries an OUTFILE keyword-// token. Doris only; see validateQuery for why the AST cannot answer this.-//-// The lexer decides, not the text, so OUTFILE inside a string literal, a-// backtick-quoted identifier or a plain comment yields no keyword and the-// statement serves. Reading an audit log for attempted exports stays a-// servable query.-//-// Nested block comments are the exception, and they fail closed rather than-// being understood: this lexer ends a comment at the first "*/", so-// "/* a /* b */ OUTFILE */" leaks its tail back into the token stream and the-// keyword is seen. Doris itself nests, so the two disagree about what such a-// statement even is (BOT-93). The disagreement is one-directional here: what-// leaks is extra tokens, so a write Doris would run cannot go unseen.-//-// A statement that will not lex fails closed as well: an unterminated string-// or comment is exactly where a keyword could be hiding.-func mentionsFileExport(statement string) bool {- tokens, lexErrors := parser.Tokenize(statement)- if len(lexErrors) > 0 {- return true- }- for _, token := range tokens {- if token.Kind == outfileTokenKind {- return true- }- }- return false-}--// outfileTokenKind is resolved from the keyword table rather than hardcoded.-// DUMPFILE is deliberately absent: it is MySQL syntax, not a Doris keyword.-var outfileTokenKind, _ = parser.KeywordToken("outfile")diff --git a/backend/plugin/parser/doris/query_test.go b/backend/plugin/parser/doris/query_test.goindex c1e88fc553..cd9cb0a9e3 100644--- a/backend/plugin/parser/doris/query_test.go+++ b/backend/plugin/parser/doris/query_test.go@@ -315,3 +315,63 @@ func TestValidateQuerySetStatements(t *testing.T) {})}}++// TestValidateQueryFileExport pins the INTO OUTFILE gate. The clause writes+// the result set to S3 or HDFS, so an export is not a read however it is+// spelled — and the keyword appearing in a string, a quoted identifier or a+// comment is not an export at all. Both halves are read off the AST, which+// is what omni's Doris grammar gained the clause for (BOT-92).+func TestValidateQueryFileExport(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ wantValid bool+ }{+ // Every position the clause can occupy: it attaches to whichever+ // node is the statement's outermost query.+ {"plain", "SELECT 1 INTO OUTFILE '/tmp/x'", false},+ {"s3 with format and properties", `SELECT * FROM t INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "example.com")`, false},+ {"after ORDER BY and LIMIT", "SELECT * FROM t ORDER BY a LIMIT 10 INTO OUTFILE '/tmp/x'", false},+ {"on a set operation", "SELECT a FROM t1 UNION SELECT a FROM t2 INTO OUTFILE '/tmp/x'", false},+ {"after parentheses", "(SELECT * FROM t) INTO OUTFILE '/tmp/x'", false},+ {"on a grouped query", "(SELECT 1 ORDER BY 1) ORDER BY 1 INTO OUTFILE '/tmp/x'", false},+ {"with a CTE", "WITH c AS (SELECT 1 AS a) SELECT * FROM c INTO OUTFILE '/tmp/x'", false},+ // The engine fails EXPLAIN of an export with the same broker/path+ // error as the export itself, so the clause is not inert under it.+ {"under EXPLAIN", "EXPLAIN SELECT 1 INTO OUTFILE '/tmp/x'", false},+ {"beside a read in one script", "SELECT 1; SELECT 2 INTO OUTFILE '/tmp/x'", false},++ // The keyword where it is not a clause. A text or token match could+ // not tell these from the real thing; the tree can.+ {"in a string literal", `SELECT a FROM t WHERE a = 'into outfile'`, true},+ {"in a quoted identifier", "SELECT `INTO OUTFILE` FROM t", true},+ {"in a block comment", "SELECT * FROM t /* INTO OUTFILE */", true},+ {"in a line comment", "SELECT * FROM t -- INTO OUTFILE 's3://bucket/'", true},+ {"a plain read", "SELECT * FROM t", true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ valid, _, err := validateQuery(tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.wantValid, valid)+ })+ }++ // MySQL's INTO DUMPFILE and INTO @var are not Doris syntax: they never+ // reach the gate, they fail to parse.+ for _, statement := range []string{+ "SELECT 1 INTO DUMPFILE '/tmp/x'",+ "SELECT 1 INTO @v",+ // The clause cannot appear inside a query, only after one.+ "SELECT * FROM (SELECT a FROM t INTO OUTFILE '/tmp/x') x",+ "SELECT 1 UNION (SELECT a FROM t INTO OUTFILE '/tmp/x')",+ // A statement that will not lex was refused as a possible export+ // while the gate read tokens; it is now reported as the syntax+ // error it is.+ "SELECT 'unterminated",+ } {+ t.Run("syntax error: "+statement, func(t *testing.T) {+ _, _, err := validateQuery(statement)+ require.Error(t, err)+ })+ }+}diff --git a/backend/plugin/parser/starrocks/query.go b/backend/plugin/parser/starrocks/query.goindex beb39b5e2c..218b56cc60 100644--- a/backend/plugin/parser/starrocks/query.go+++ b/backend/plugin/parser/starrocks/query.go@@ -75,15 +75,8 @@ func needsExecute(node ast.Node) bool {// SQL after parseStarRocksSQL succeeds.func isReadOnlyAST(node ast.Node) bool {switch n := node.(type) {- case *ast.SelectStmt:- // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.- return n.Into == nil- case *ast.SetOpStmt:- return !setOpWritesOutfile(n)- case *ast.ParenSelect:- // (SELECT ...) at the top level — classify by the inner query so the- // INTO OUTFILE gate still applies inside the parens.- return isReadOnlyAST(n.Sel)+ case *ast.SelectStmt, *ast.SetOpStmt, *ast.ParenSelect:+ return isReadOnlyQuery(node)case *ast.ShowStmt:// Reject bare `SHOW` (Type is empty when the parser took the stub path// without seeing a recognised variant keyword).@@ -97,9 +90,20 @@ func isReadOnlyAST(node ast.Node) bool {return n.Target != nilcase *ast.ExplainStmt:// EXPLAIN requires an inner query; the inner query must itself be a- // shape EXPLAIN is meaningful for (SELECT/DML/Show). This blocks+ // shape EXPLAIN is meaningful for (SELECT family or DML). This blocks// inputs like `EXPLAIN`, `EXPLAIN DROP TABLE t`, etc.- return n.Query != nil && isExplainableInner(n.Query)+ if n.Query == nil || !isExplainableInner(n.Query) {+ return false+ }+ // EXPLAIN ANALYZE runs the statement rather than only planning it, so+ // over DML it writes. StarRocks aborts the INSERT transaction by+ // default, but the run still creates any partitions the rows need and+ // those outlive the abort. The plan-only modifiers never execute, so+ // they stay readable over DML.+ if n.Type == "ANALYZE" {+ return isReadOnlyQuery(n.Query)+ }+ return truecase *ast.HelpStmt:return truedefault:@@ -109,15 +113,13 @@ func isReadOnlyAST(node ast.Node) bool {// isExplainableInner reports whether `node` is a shape that EXPLAIN can// legitimately wrap (SELECT family or DML). DDL inside EXPLAIN is rejected:-// Doris only supports EXPLAIN on query and DML statements.+// StarRocks only supports EXPLAIN on query and DML statements. Queries are+// classified by isReadOnlyQuery, the one export gate.func isExplainableInner(node ast.Node) bool {- switch n := node.(type) {- case *ast.SelectStmt, *ast.SetOpStmt:+ if isReadOnlyQuery(node) {return true- case *ast.ParenSelect:- // EXPLAIN (SELECT ...) / EXPLAIN ((SELECT ...)) — engine-verified- // accepts; classify by the wrapped query.- return isExplainableInner(n.Sel)+ }+ switch node.(type) {case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt,*ast.MergeStmt, *ast.TruncateTableStmt:return true@@ -125,6 +127,30 @@ func isExplainableInner(node ast.Node) bool {return false}+// isReadOnlyQuery reports whether `node` is a SELECT-family query that reads+// and never writes. INTO OUTFILE exports to S3 or HDFS, past masking and past+// the export gate, so a query carrying one is not a read.+//+// This is the single export gate for the SELECT family. Both the top level+// (isReadOnlyAST) and the EXPLAIN path (isExplainableInner) go through it, so+// the editor cannot accept by way of EXPLAIN what it refuses when asked+// directly.+func isReadOnlyQuery(node ast.Node) bool {+ switch n := node.(type) {+ case *ast.SelectStmt:+ return n.Into == nil+ case *ast.SetOpStmt:+ // The parser attaches INTO to an arm rather than the set operation.+ return !setOpWritesOutfile(n)+ case *ast.ParenSelect:+ // (SELECT ...) / ((SELECT ...)) — classify by the wrapped query so the+ // gate still applies inside the parens. ParenSelect only ever wraps a+ // SELECT, SetOpStmt or another ParenSelect, so no DML hides here.+ return isReadOnlyQuery(n.Sel)+ }+ return false+}+// setOpWritesOutfile walks the arms: the parser attaches INTO to an arm rather// than to the set operation, and arms nest.func setOpWritesOutfile(node ast.Node) bool {diff --git a/backend/plugin/parser/starrocks/query_test.go b/backend/plugin/parser/starrocks/query_test.goindex 6daadabd31..85e3f1a133 100644--- a/backend/plugin/parser/starrocks/query_test.go+++ b/backend/plugin/parser/starrocks/query_test.go@@ -142,6 +142,93 @@ func TestValidateQuery(t *testing.T) {valid: true,description: "EXPLAIN over nested parens is read-only",},+ {+ // Plain EXPLAIN only plans, so it stays read-only over DML (the+ // cases above). EXPLAIN ANALYZE runs the statement instead, which+ // over DML writes: StarRocks aborts the INSERT transaction but the+ // run still creates any partitions the rows need.+ statement: "EXPLAIN ANALYZE SELECT a FROM t",+ valid: true,+ description: "EXPLAIN ANALYZE over a SELECT is read-only",+ },+ {+ statement: "EXPLAIN ANALYZE (SELECT 1)",+ valid: true,+ description: "EXPLAIN ANALYZE over a parenthesized query is read-only",+ },+ {+ statement: "EXPLAIN ANALYZE ((SELECT 1))",+ valid: true,+ description: "EXPLAIN ANALYZE over nested parens is read-only",+ },+ {+ statement: "EXPLAIN ANALYZE SELECT 1 UNION SELECT 2",+ valid: true,+ description: "EXPLAIN ANALYZE over a set operation is read-only",+ },+ {+ statement: "EXPLAIN VERBOSE INSERT INTO users (id, name) VALUES (1, 'test')",+ valid: true,+ description: "a plan-only modifier stays read-only over DML",+ },+ {+ statement: "EXPLAIN ANALYZE INSERT INTO users (id, name) VALUES (1, 'test')",+ valid: false,+ description: "EXPLAIN ANALYZE over INSERT must be rejected",+ },+ {+ statement: "EXPLAIN ANALYZE INSERT INTO users SELECT id, name FROM other",+ valid: false,+ description: "EXPLAIN ANALYZE over INSERT ... SELECT must be rejected",+ },+ {+ statement: "EXPLAIN ANALYZE UPDATE users SET name = 'test' WHERE id = 1",+ valid: false,+ description: "EXPLAIN ANALYZE over UPDATE must be rejected",+ },+ {+ statement: "EXPLAIN ANALYZE DELETE FROM users WHERE id = 1",+ valid: false,+ description: "EXPLAIN ANALYZE over DELETE must be rejected",+ },+ {+ statement: "EXPLAIN ANALYZE TRUNCATE TABLE users",+ valid: false,+ description: "EXPLAIN ANALYZE over TRUNCATE must be rejected",+ },+ {+ // Plain EXPLAIN does not execute, but the editor must refuse by way+ // of EXPLAIN what it refuses when asked directly — and under ANALYZE,+ // which does execute, the export would be live.+ statement: `EXPLAIN SELECT a FROM t INTO OUTFILE "s3://b/x"`,+ valid: false,+ description: "EXPLAIN must not launder an export past the gate",+ },+ {+ statement: `EXPLAIN VERBOSE SELECT a FROM t INTO OUTFILE "s3://b/x"`,+ valid: false,+ description: "a plan-only modifier must not launder an export either",+ },+ {+ statement: `EXPLAIN ANALYZE SELECT a FROM t INTO OUTFILE "s3://b/x"`,+ valid: false,+ description: "EXPLAIN ANALYZE over an export must be rejected",+ },+ {+ statement: `EXPLAIN ANALYZE (SELECT a FROM t INTO OUTFILE "s3://b/x")`,+ valid: false,+ description: "EXPLAIN ANALYZE over a parenthesized export must be rejected",+ },+ {+ statement: `EXPLAIN ANALYZE ((SELECT a FROM t INTO OUTFILE "s3://b/x"))`,+ valid: false,+ description: "EXPLAIN ANALYZE over a nested parenthesized export must be rejected",+ },+ {+ statement: `EXPLAIN ANALYZE SELECT 1 UNION SELECT a FROM t INTO OUTFILE "s3://b/x"`,+ valid: false,+ description: "EXPLAIN ANALYZE over an export in a set-operation arm must be rejected",+ },{statement: "SELECT EXTRACT(YEAR FROM MONTHS_ADD(NOW(), -1))",valid: true,