Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(doris,starrocks): gate INTO OUTFILE and EXPLAIN ANALYZE off the AST (#21735)

Danny Xue26dc2d3bb61
diff --git a/backend/api/v1/mcp_sql_clamp_test.go b/backend/api/v1/mcp_sql_clamp_test.go
index 36e428630d..a1b4479800 100644
--- a/backend/api/v1/mcp_sql_clamp_test.go
+++ b/backend/api/v1/mcp_sql_clamp_test.go
@@ -44,7 +44,7 @@ import (
// unclassifiedEngineWrites are the engines with no registered read-only
// classifier, each with a statement that writes on it. Every one fails closed
-// under the clamp rather than gaining a validator in this change.
+// under the clamp rather than gaining a validator.
//
// One map rather than a list beside a lookup table: two copies of this set
// could disagree about which engines the test is even about.
@@ -269,15 +269,18 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {
reason: "the statement is not a read",
},
{
- // Doris nests block comments; the shared stripper stops at the
- // first "*/", so stripping first would leave text between INTO
- // and OUTFILE and lose a keyword-pair match. The guard reads the
- // raw statement for that reason.
+ // Doris nests block comments and omni's Doris lexer does not: it
+ // ends one at the first "*/", so the tail leaks back into the
+ // token stream. What leaks is not a clause, so the statement does
+ // not parse and is refused as a syntax error rather than reaching
+ // the export gate. Doris, which nests, reads the same text as an
+ // export, so the two disagree about what this statement is and it
+ // is refused either way (BOT-93).
name: "a nested comment cannot hide a Doris export",
engine: storepb.Engine_DORIS,
statement: "SELECT * FROM employee\nINTO /* outer /* inner */ tail */ OUTFILE \"s3://bucket/export/\"\nFORMAT AS PARQUET",
refused: true,
- reason: "the statement is not a read",
+ reason: "the statement could not be parsed",
},
{
name: "nor can a line comment",
@@ -287,11 +290,16 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {
reason: "the statement is not a read",
},
{
+ // An unterminated literal means there is no tree to classify, so
+ // the clamp refuses it as the syntax error it is. It was refused
+ // as "not a read" while a token scan stood in for the export
+ // gate, since that scan treated any lex error as a possible
+ // hidden keyword.
name: "an unlexable Doris statement fails closed",
engine: storepb.Engine_DORIS,
statement: "SELECT * FROM t WHERE a = 'unterminated",
refused: true,
- reason: "the statement is not a read",
+ reason: "the statement could not be parsed",
},
{
// USE rebinds the connection's catalog and schema, so an
@@ -349,16 +357,17 @@ func TestMCPClampRefusesWhatItCannotShowIsARead(t *testing.T) {
},
{
// Not "inside a comment" for a NESTED one: omni's Doris lexer ends
- // a comment at the first "*/", so the tail leaks back into the
- // token stream and the keyword is seen. Doris itself nests, so
- // this query is a plain SELECT 1 there and Bytebase refuses it
- // anyway (BOT-93). Pinned as the cost of failing closed on a
- // disagreement rather than guessing which lexer is right.
+ // a comment at the first "*/", so "OUTFILE */" leaks back into the
+ // token stream, where it parses as nothing and the statement is
+ // refused as a syntax error. Doris itself nests, so this query is
+ // a plain SELECT 1 there (BOT-93). Pinned as the cost of failing
+ // closed on a disagreement rather than guessing which lexer is
+ // right.
name: "a nested comment holding the keyword refuses, though Doris would not",
engine: storepb.Engine_DORIS,
statement: "SELECT 1 /* outer /* inner */ OUTFILE */",
refused: true,
- reason: "the statement is not a read",
+ reason: "the statement could not be parsed",
},
{
name: "a read on an engine with no splitter is still classified",
diff --git a/backend/plugin/parser/doris/limit.go b/backend/plugin/parser/doris/limit.go
index 07935e365a..c256641d69 100644
--- a/backend/plugin/parser/doris/limit.go
+++ b/backend/plugin/parser/doris/limit.go
@@ -42,14 +42,21 @@ func statementWithResultLimit(statement string, limit int, _ string) string {
return stmt
}
-// The Doris AST cannot represent INTO OUTFILE. Keep this top-level clause outside
-// the query while capping it, including when a set operation needs wrapping.
+// splitOutfileClause cuts a top-level INTO OUTFILE clause off the end of the
+// statement, so the clause stays outside the query while it is capped,
+// including when a set operation needs wrapping.
+//
+// This reads tokens rather than the tree the clause now has (omni's
+// doris/ast.IntoOutfileClause) because it has to answer for text that does
+// not parse: a statement whose parse fails still reaches the caller's
+// CTE-wrap fallback, which needs the clause held back from the wrap.
func splitOutfileClause(statement string) (string, string) {
tokens, errs := parser.Tokenize(statement)
if len(errs) > 0 {
return statement, ""
}
intoKind, _ := parser.KeywordToken("into")
+ outfileKind, _ := parser.KeywordToken("outfile")
depth := 0
for i, token := range tokens {
switch token.Kind {
@@ -60,7 +67,7 @@ func splitOutfileClause(statement string) (string, string) {
case ';':
return statement, ""
case intoKind:
- if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileTokenKind {
+ if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileKind {
return statement[:token.Loc.Start], statement[token.Loc.Start:]
}
default:
diff --git a/backend/plugin/parser/doris/query.go b/backend/plugin/parser/doris/query.go
index 0e35cfa64a..12b5836223 100644
--- a/backend/plugin/parser/doris/query.go
+++ b/backend/plugin/parser/doris/query.go
@@ -2,7 +2,6 @@ package doris
import (
"github.com/bytebase/omni/doris/ast"
- "github.com/bytebase/omni/doris/parser"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -20,6 +19,11 @@ func init() {
// alone would mis-accept CTE-prefixed DML such as `WITH x AS (...) UPDATE ...`,
// which Classify would tag as SELECT because `WITH` is its first token.
//
+// A query that carries INTO OUTFILE is refused: it writes its result set to
+// S3 or HDFS, so the data leaves the database past masking and past the
+// export gate. isReadOnlyAST reads the clause off the tree, so the words in a
+// comment or a quoted identifier are not mistaken for one (BOT-92).
+//
// Syntax errors are surfaced up; that lets validateQueryRequest reject
// malformed read-only SQL before execution.
//
@@ -28,14 +32,6 @@ func init() {
// false once a statement needs executing rather than reading, which is how the
// driver decides between QueryContext and ExecContext.
func validateQuery(statement string) (bool, bool, error) {
- // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.
- // Matched on text because omni's Doris grammar has no INTO node: it parses
- // to a bare SelectStmt identical to one without the clause (BOT-92).
- // Replace with an AST check, as starrocks/query.go does, once it has one.
- //
- if mentionsFileExport(statement) {
- return false, false, nil
- }
parsed, err := parseDorisSQL(statement)
if err != nil {
return false, false, err
@@ -84,12 +80,17 @@ func needsExecute(node ast.Node) bool {
// SQL after parseDorisSQL succeeds.
func isReadOnlyAST(node ast.Node) bool {
switch n := node.(type) {
- case *ast.SelectStmt, *ast.SetOpStmt:
- return true
+ case *ast.SelectStmt:
+ // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.
+ return n.Into == nil
+ case *ast.SetOpStmt:
+ return n.Into == nil
case *ast.GroupedQuery:
// A repeated trailing clause group or a parenthesized WITH group —
// (SELECT 1 LIMIT 1) LIMIT 2 — wraps a query; classify by the query.
- return isReadOnlyAST(n.Query)
+ // The export gate applies to the group as well, since the clause
+ // attaches to whichever node is the statement's outermost query.
+ return n.Into == nil && isReadOnlyAST(n.Query)
case *ast.ShowStmt:
// Reject bare `SHOW` (Type is empty when the parser took the stub path
// without seeing a recognised variant keyword).
@@ -116,51 +117,25 @@ func isReadOnlyAST(node ast.Node) bool {
// isExplainableInner reports whether `node` is a shape that EXPLAIN can
// legitimately wrap (SELECT family or DML). DDL inside EXPLAIN is rejected:
// Doris only supports EXPLAIN on query and DML statements.
+//
+// An export stays refused under EXPLAIN. The engine does not treat
+// `EXPLAIN SELECT ... INTO OUTFILE ...` as a plain plan: it fails with the
+// same "Please specify BROKER properties or check your local file path" as
+// the bare export, where `EXPLAIN SELECT 1` returns a plan (container-
+// verified). So the clause is not inert here, and the gate holds.
func isExplainableInner(node ast.Node) bool {
switch n := node.(type) {
- case *ast.SelectStmt, *ast.SetOpStmt:
- return true
+ case *ast.SelectStmt:
+ return n.Into == nil
+ case *ast.SetOpStmt:
+ return n.Into == nil
case *ast.GroupedQuery:
// EXPLAIN (SELECT 1 LIMIT 1) LIMIT 2 — engine-verified accept;
// classify by the wrapped query.
- return isExplainableInner(n.Query)
+ return n.Into == nil && isExplainableInner(n.Query)
case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt,
*ast.MergeStmt, *ast.TruncateTableStmt:
return true
}
return false
}
-
-// mentionsFileExport reports whether the statement carries an OUTFILE keyword
-// token. Doris only; see validateQuery for why the AST cannot answer this.
-//
-// The lexer decides, not the text, so OUTFILE inside a string literal, a
-// backtick-quoted identifier or a plain comment yields no keyword and the
-// statement serves. Reading an audit log for attempted exports stays a
-// servable query.
-//
-// Nested block comments are the exception, and they fail closed rather than
-// being understood: this lexer ends a comment at the first "*/", so
-// "/* a /* b */ OUTFILE */" leaks its tail back into the token stream and the
-// keyword is seen. Doris itself nests, so the two disagree about what such a
-// statement even is (BOT-93). The disagreement is one-directional here: what
-// leaks is extra tokens, so a write Doris would run cannot go unseen.
-//
-// A statement that will not lex fails closed as well: an unterminated string
-// or comment is exactly where a keyword could be hiding.
-func mentionsFileExport(statement string) bool {
- tokens, lexErrors := parser.Tokenize(statement)
- if len(lexErrors) > 0 {
- return true
- }
- for _, token := range tokens {
- if token.Kind == outfileTokenKind {
- return true
- }
- }
- return false
-}
-
-// outfileTokenKind is resolved from the keyword table rather than hardcoded.
-// DUMPFILE is deliberately absent: it is MySQL syntax, not a Doris keyword.
-var outfileTokenKind, _ = parser.KeywordToken("outfile")
diff --git a/backend/plugin/parser/doris/query_test.go b/backend/plugin/parser/doris/query_test.go
index c1e88fc553..cd9cb0a9e3 100644
--- a/backend/plugin/parser/doris/query_test.go
+++ b/backend/plugin/parser/doris/query_test.go
@@ -315,3 +315,63 @@ func TestValidateQuerySetStatements(t *testing.T) {
})
}
}
+
+// TestValidateQueryFileExport pins the INTO OUTFILE gate. The clause writes
+// the result set to S3 or HDFS, so an export is not a read however it is
+// spelled — and the keyword appearing in a string, a quoted identifier or a
+// comment is not an export at all. Both halves are read off the AST, which
+// is what omni's Doris grammar gained the clause for (BOT-92).
+func TestValidateQueryFileExport(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ wantValid bool
+ }{
+ // Every position the clause can occupy: it attaches to whichever
+ // node is the statement's outermost query.
+ {"plain", "SELECT 1 INTO OUTFILE '/tmp/x'", false},
+ {"s3 with format and properties", `SELECT * FROM t INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "example.com")`, false},
+ {"after ORDER BY and LIMIT", "SELECT * FROM t ORDER BY a LIMIT 10 INTO OUTFILE '/tmp/x'", false},
+ {"on a set operation", "SELECT a FROM t1 UNION SELECT a FROM t2 INTO OUTFILE '/tmp/x'", false},
+ {"after parentheses", "(SELECT * FROM t) INTO OUTFILE '/tmp/x'", false},
+ {"on a grouped query", "(SELECT 1 ORDER BY 1) ORDER BY 1 INTO OUTFILE '/tmp/x'", false},
+ {"with a CTE", "WITH c AS (SELECT 1 AS a) SELECT * FROM c INTO OUTFILE '/tmp/x'", false},
+ // The engine fails EXPLAIN of an export with the same broker/path
+ // error as the export itself, so the clause is not inert under it.
+ {"under EXPLAIN", "EXPLAIN SELECT 1 INTO OUTFILE '/tmp/x'", false},
+ {"beside a read in one script", "SELECT 1; SELECT 2 INTO OUTFILE '/tmp/x'", false},
+
+ // The keyword where it is not a clause. A text or token match could
+ // not tell these from the real thing; the tree can.
+ {"in a string literal", `SELECT a FROM t WHERE a = 'into outfile'`, true},
+ {"in a quoted identifier", "SELECT `INTO OUTFILE` FROM t", true},
+ {"in a block comment", "SELECT * FROM t /* INTO OUTFILE */", true},
+ {"in a line comment", "SELECT * FROM t -- INTO OUTFILE 's3://bucket/'", true},
+ {"a plain read", "SELECT * FROM t", true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ valid, _, err := validateQuery(tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, tc.wantValid, valid)
+ })
+ }
+
+ // MySQL's INTO DUMPFILE and INTO @var are not Doris syntax: they never
+ // reach the gate, they fail to parse.
+ for _, statement := range []string{
+ "SELECT 1 INTO DUMPFILE '/tmp/x'",
+ "SELECT 1 INTO @v",
+ // The clause cannot appear inside a query, only after one.
+ "SELECT * FROM (SELECT a FROM t INTO OUTFILE '/tmp/x') x",
+ "SELECT 1 UNION (SELECT a FROM t INTO OUTFILE '/tmp/x')",
+ // A statement that will not lex was refused as a possible export
+ // while the gate read tokens; it is now reported as the syntax
+ // error it is.
+ "SELECT 'unterminated",
+ } {
+ t.Run("syntax error: "+statement, func(t *testing.T) {
+ _, _, err := validateQuery(statement)
+ require.Error(t, err)
+ })
+ }
+}
diff --git a/backend/plugin/parser/starrocks/query.go b/backend/plugin/parser/starrocks/query.go
index beb39b5e2c..218b56cc60 100644
--- a/backend/plugin/parser/starrocks/query.go
+++ b/backend/plugin/parser/starrocks/query.go
@@ -75,15 +75,8 @@ func needsExecute(node ast.Node) bool {
// SQL after parseStarRocksSQL succeeds.
func isReadOnlyAST(node ast.Node) bool {
switch n := node.(type) {
- case *ast.SelectStmt:
- // INTO OUTFILE exports to S3 or HDFS, past masking and the export gate.
- return n.Into == nil
- case *ast.SetOpStmt:
- return !setOpWritesOutfile(n)
- case *ast.ParenSelect:
- // (SELECT ...) at the top level — classify by the inner query so the
- // INTO OUTFILE gate still applies inside the parens.
- return isReadOnlyAST(n.Sel)
+ case *ast.SelectStmt, *ast.SetOpStmt, *ast.ParenSelect:
+ return isReadOnlyQuery(node)
case *ast.ShowStmt:
// Reject bare `SHOW` (Type is empty when the parser took the stub path
// without seeing a recognised variant keyword).
@@ -97,9 +90,20 @@ func isReadOnlyAST(node ast.Node) bool {
return n.Target != nil
case *ast.ExplainStmt:
// EXPLAIN requires an inner query; the inner query must itself be a
- // shape EXPLAIN is meaningful for (SELECT/DML/Show). This blocks
+ // shape EXPLAIN is meaningful for (SELECT family or DML). This blocks
// inputs like `EXPLAIN`, `EXPLAIN DROP TABLE t`, etc.
- return n.Query != nil && isExplainableInner(n.Query)
+ if n.Query == nil || !isExplainableInner(n.Query) {
+ return false
+ }
+ // EXPLAIN ANALYZE runs the statement rather than only planning it, so
+ // over DML it writes. StarRocks aborts the INSERT transaction by
+ // default, but the run still creates any partitions the rows need and
+ // those outlive the abort. The plan-only modifiers never execute, so
+ // they stay readable over DML.
+ if n.Type == "ANALYZE" {
+ return isReadOnlyQuery(n.Query)
+ }
+ return true
case *ast.HelpStmt:
return true
default:
@@ -109,15 +113,13 @@ func isReadOnlyAST(node ast.Node) bool {
// isExplainableInner reports whether `node` is a shape that EXPLAIN can
// legitimately wrap (SELECT family or DML). DDL inside EXPLAIN is rejected:
-// Doris only supports EXPLAIN on query and DML statements.
+// StarRocks only supports EXPLAIN on query and DML statements. Queries are
+// classified by isReadOnlyQuery, the one export gate.
func isExplainableInner(node ast.Node) bool {
- switch n := node.(type) {
- case *ast.SelectStmt, *ast.SetOpStmt:
+ if isReadOnlyQuery(node) {
return true
- case *ast.ParenSelect:
- // EXPLAIN (SELECT ...) / EXPLAIN ((SELECT ...)) — engine-verified
- // accepts; classify by the wrapped query.
- return isExplainableInner(n.Sel)
+ }
+ switch node.(type) {
case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt,
*ast.MergeStmt, *ast.TruncateTableStmt:
return true
@@ -125,6 +127,30 @@ func isExplainableInner(node ast.Node) bool {
return false
}
+// isReadOnlyQuery reports whether `node` is a SELECT-family query that reads
+// and never writes. INTO OUTFILE exports to S3 or HDFS, past masking and past
+// the export gate, so a query carrying one is not a read.
+//
+// This is the single export gate for the SELECT family. Both the top level
+// (isReadOnlyAST) and the EXPLAIN path (isExplainableInner) go through it, so
+// the editor cannot accept by way of EXPLAIN what it refuses when asked
+// directly.
+func isReadOnlyQuery(node ast.Node) bool {
+ switch n := node.(type) {
+ case *ast.SelectStmt:
+ return n.Into == nil
+ case *ast.SetOpStmt:
+ // The parser attaches INTO to an arm rather than the set operation.
+ return !setOpWritesOutfile(n)
+ case *ast.ParenSelect:
+ // (SELECT ...) / ((SELECT ...)) — classify by the wrapped query so the
+ // gate still applies inside the parens. ParenSelect only ever wraps a
+ // SELECT, SetOpStmt or another ParenSelect, so no DML hides here.
+ return isReadOnlyQuery(n.Sel)
+ }
+ return false
+}
+
// setOpWritesOutfile walks the arms: the parser attaches INTO to an arm rather
// than to the set operation, and arms nest.
func setOpWritesOutfile(node ast.Node) bool {
diff --git a/backend/plugin/parser/starrocks/query_test.go b/backend/plugin/parser/starrocks/query_test.go
index 6daadabd31..85e3f1a133 100644
--- a/backend/plugin/parser/starrocks/query_test.go
+++ b/backend/plugin/parser/starrocks/query_test.go
@@ -142,6 +142,93 @@ func TestValidateQuery(t *testing.T) {
valid: true,
description: "EXPLAIN over nested parens is read-only",
},
+ {
+ // Plain EXPLAIN only plans, so it stays read-only over DML (the
+ // cases above). EXPLAIN ANALYZE runs the statement instead, which
+ // over DML writes: StarRocks aborts the INSERT transaction but the
+ // run still creates any partitions the rows need.
+ statement: "EXPLAIN ANALYZE SELECT a FROM t",
+ valid: true,
+ description: "EXPLAIN ANALYZE over a SELECT is read-only",
+ },
+ {
+ statement: "EXPLAIN ANALYZE (SELECT 1)",
+ valid: true,
+ description: "EXPLAIN ANALYZE over a parenthesized query is read-only",
+ },
+ {
+ statement: "EXPLAIN ANALYZE ((SELECT 1))",
+ valid: true,
+ description: "EXPLAIN ANALYZE over nested parens is read-only",
+ },
+ {
+ statement: "EXPLAIN ANALYZE SELECT 1 UNION SELECT 2",
+ valid: true,
+ description: "EXPLAIN ANALYZE over a set operation is read-only",
+ },
+ {
+ statement: "EXPLAIN VERBOSE INSERT INTO users (id, name) VALUES (1, 'test')",
+ valid: true,
+ description: "a plan-only modifier stays read-only over DML",
+ },
+ {
+ statement: "EXPLAIN ANALYZE INSERT INTO users (id, name) VALUES (1, 'test')",
+ valid: false,
+ description: "EXPLAIN ANALYZE over INSERT must be rejected",
+ },
+ {
+ statement: "EXPLAIN ANALYZE INSERT INTO users SELECT id, name FROM other",
+ valid: false,
+ description: "EXPLAIN ANALYZE over INSERT ... SELECT must be rejected",
+ },
+ {
+ statement: "EXPLAIN ANALYZE UPDATE users SET name = 'test' WHERE id = 1",
+ valid: false,
+ description: "EXPLAIN ANALYZE over UPDATE must be rejected",
+ },
+ {
+ statement: "EXPLAIN ANALYZE DELETE FROM users WHERE id = 1",
+ valid: false,
+ description: "EXPLAIN ANALYZE over DELETE must be rejected",
+ },
+ {
+ statement: "EXPLAIN ANALYZE TRUNCATE TABLE users",
+ valid: false,
+ description: "EXPLAIN ANALYZE over TRUNCATE must be rejected",
+ },
+ {
+ // Plain EXPLAIN does not execute, but the editor must refuse by way
+ // of EXPLAIN what it refuses when asked directly — and under ANALYZE,
+ // which does execute, the export would be live.
+ statement: `EXPLAIN SELECT a FROM t INTO OUTFILE "s3://b/x"`,
+ valid: false,
+ description: "EXPLAIN must not launder an export past the gate",
+ },
+ {
+ statement: `EXPLAIN VERBOSE SELECT a FROM t INTO OUTFILE "s3://b/x"`,
+ valid: false,
+ description: "a plan-only modifier must not launder an export either",
+ },
+ {
+ statement: `EXPLAIN ANALYZE SELECT a FROM t INTO OUTFILE "s3://b/x"`,
+ valid: false,
+ description: "EXPLAIN ANALYZE over an export must be rejected",
+ },
+ {
+ statement: `EXPLAIN ANALYZE (SELECT a FROM t INTO OUTFILE "s3://b/x")`,
+ valid: false,
+ description: "EXPLAIN ANALYZE over a parenthesized export must be rejected",
+ },
+ {
+ statement: `EXPLAIN ANALYZE ((SELECT a FROM t INTO OUTFILE "s3://b/x"))`,
+ valid: false,
+ description: "EXPLAIN ANALYZE over a nested parenthesized export must be rejected",
+ },
+ {
+ statement: `EXPLAIN ANALYZE SELECT 1 UNION SELECT a FROM t INTO OUTFILE "s3://b/x"`,
+ valid: false,
+ description: "EXPLAIN ANALYZE over an export in a set-operation arm must be rejected",
+ },
{
statement: "SELECT EXTRACT(YEAR FROM MONTHS_ADD(NOW(), -1))",
valid: true,