Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(explain): stop reading a statement's first keyword past a nested comment (#21689)
Danny Xu709486746633
diff --git a/backend/plugin/parser/base/explain.go b/backend/plugin/parser/base/explain.goindex a35c5857d3..f004f04a1e 100644--- a/backend/plugin/parser/base/explain.go+++ b/backend/plugin/parser/base/explain.go@@ -93,6 +93,16 @@ var ErrExplainAnalyzeExecutes = errors.New("EXPLAIN ANALYZE executes the stateme// ignoring leading comments and whitespace. It stands in for an AST check on an// engine whose parser cannot answer, which is enough to keep such an engine from// planning a plan.+//+// A block comment holding another one is read as no keyword at all, because+// which statement follows it depends on the engine and this function has none+// to ask. PostgreSQL, SQL Server, Snowflake and Spark nest them, so in+// `/* /* */ EXPLAIN */ SELECT ssn FROM t` the EXPLAIN is still inside the+// comment and the statement is the SELECT. MySQL and Oracle end the comment at+// the first `*/`, so in `/* /* */ SELECT '*/ EXPLAIN', ssn FROM t` the+// statement is that SELECT and the EXPLAIN is inside a string. Either reading+// hands the other dialect a statement that returns rows, and callers stand work+// down for a plan — masking is one — so neither is safe to guess.func StartsWithExplain(statement string) bool {s := statementfor {@@ -105,8 +115,8 @@ func StartsWithExplain(statement string) bool {}s = restcase strings.HasPrefix(s, "/*"):- _, rest, found := strings.Cut(s[2:], "*/")- if !found {+ rest, closed := pastBlockComment(s)+ if !closed {return false}s = rest@@ -125,3 +135,23 @@ func StartsWithExplain(statement string) bool {}}}++// pastBlockComment returns what follows the block comment opening s, and+// reports whether it could be read at all. A comment left open cannot be, and+// neither can one holding another `/*`: see StartsWithExplain for why that is+// a question about the engine rather than about the text.+//+// Whichever delimiter comes first decides, one position at a time, because the+// two can overlap: the second `/*` of `/*/*/` ends where the first `*/` begins,+// so searching for the closer and then looking back for an opener finds none.+func pastBlockComment(s string) (string, bool) {+ for at := len("/*"); at < len(s); at++ {+ switch {+ case strings.HasPrefix(s[at:], "/*"):+ return "", false+ case strings.HasPrefix(s[at:], "*/"):+ return s[at+len("*/"):], true+ }+ }+ return "", false+}diff --git a/backend/plugin/parser/base/explain_test.go b/backend/plugin/parser/base/explain_test.goindex 48e7363ae8..be76ed680e 100644--- a/backend/plugin/parser/base/explain_test.go+++ b/backend/plugin/parser/base/explain_test.go@@ -26,6 +26,22 @@ func TestStartsWithExplain(t *testing.T) {// A comment that never ends leaves no keyword to read.{"/* EXPLAIN SELECT 1", false},{"-- EXPLAIN SELECT 1", false},+ // A comment holding another one reads differently on an engine that+ // nests them and on one that does not, and each reading leaves the+ // other a statement that returns rows. Both of these run as a SELECT of+ // ssn — the first on PostgreSQL, the second on MySQL.+ {"/* /* */ EXPLAIN */ SELECT ssn FROM t", false},+ {`/* /* */ SELECT '*/ EXPLAIN', ssn FROM t`, false},+ // The inner comment can open where the first `*/` begins: Spark runs+ // this one as the DELETE.+ {"/*/*/ EXPLAIN */ */ DELETE FROM t", false},+ {"/* a /* b */ EXPLAIN */ DELETE FROM t", false},+ {"/*/* EXPLAIN */*/ SELECT ssn FROM t", false},+ {"/* /* unclosed */ EXPLAIN SELECT 1", false},+ {"/* a /* b */ c */ EXPLAIN SELECT 1", false},+ // A comment of its own still reads, and so does a keyword before one.+ {"/* a */ /* b */ EXPLAIN SELECT 1", true},+ {"EXPLAIN /* a /* b */ c */ SELECT 1", true},}for _, tc := range tests {require.Equalf(t, tc.want, StartsWithExplain(tc.statement), "%q", tc.statement)diff --git a/backend/plugin/parser/standard/explain.go b/backend/plugin/parser/standard/explain.goindex 0835615c3a..4928384a3f 100644--- a/backend/plugin/parser/standard/explain.go+++ b/backend/plugin/parser/standard/explain.go@@ -1,8 +1,6 @@package standardimport (- "strings"-storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -40,58 +38,15 @@ func hiveExplainStatement(statement string, format base.ExplainFormat) (string,// named; EXPLAIN never executes what it plans, so the planned form of a write// is a plan of that write.//-// Which statement that is has to be read the way Databricks reads it. Its block-// comments nest, so `/* a /* b */ EXPLAIN */ DELETE FROM t` is a DELETE with the-// EXPLAIN inside the comment, and preserving it as the caller's own plan would-// run the delete. The engines sharing this file stop at the first `*/` because a-// read-only validator is registered for them, which refuses such a statement at-// the gate; Databricks has none.+// Which statement that is has to be read the way Spark reads it, down to the+// nesting of its block comments, which is what base.StartsWithExplain reads.func databricksExplainStatement(statement string, _ base.ExplainFormat) (string, error) {- if base.StartsWithExplain(pastNestedComments(statement)) {+ if base.StartsWithExplain(statement) {return statement, nil}return "EXPLAIN FORMATTED " + statement, nil}-// pastNestedComments returns statement from its first token that is not a-// comment, counting the depth of block comments rather than ending one at the-// first `*/`. A comment left open runs to the end of the statement, which-// leaves no token at all.-func pastNestedComments(statement string) string {- s := statement- for {- s = strings.TrimLeft(s, " \t\r\n\v\f")- switch {- case strings.HasPrefix(s, "--"):- _, rest, found := strings.Cut(s, "\n")- if !found {- return ""- }- s = rest- case strings.HasPrefix(s, "/*"):- depth, at := 1, len("/*")- for depth > 0 && at < len(s) {- switch {- case strings.HasPrefix(s[at:], "/*"):- depth++- at += len("/*")- case strings.HasPrefix(s[at:], "*/"):- depth--- at += len("*/")- default:- at++- }- }- if depth > 0 {- return ""- }- s = s[at:]- default:- return s- }- }-}-// clickhouseExplainStatement asks ClickHouse for the structured plan the// query-plan viewer draws. `indexes = 1` is what makes each read report how// many parts and granules its indexes let it skip, which is what a ClickHousediff --git a/backend/plugin/parser/standard/explain_test.go b/backend/plugin/parser/standard/explain_test.goindex a50329e4af..9d4b5b3600 100644--- a/backend/plugin/parser/standard/explain_test.go+++ b/backend/plugin/parser/standard/explain_test.go@@ -102,9 +102,13 @@ func TestDatabricksExplainReadsNestedComments(t *testing.T) {"EXPLAIN FORMATTED /* outer /* inner */\nEXPLAIN\n*/\nDELETE FROM t",},{"/* a /* b */ EXPLAIN */ SELECT 1", "EXPLAIN FORMATTED /* a /* b */ EXPLAIN */ SELECT 1"},- // A nested comment that closes before a real EXPLAIN leaves the plan the- // caller asked for.- {"/* a /* b */ */ EXPLAIN SELECT 1", "/* a /* b */ */ EXPLAIN SELECT 1"},+ // A nested comment closing before a real EXPLAIN is a plan Spark would+ // have kept, and this plans it again: the keyword scan refuses to read+ // past a comment holding another one, because the same text ends in a+ // different place on an engine that does not nest them. Spark then+ // rejects `EXPLAIN FORMATTED EXPLAIN`, which costs this caller their+ // plan and costs every other engine nothing.+ {"/* a /* b */ */ EXPLAIN SELECT 1", "EXPLAIN FORMATTED /* a /* b */ */ EXPLAIN SELECT 1"},// A comment left open runs to the end, so there is no statement to// preserve.{"/* a /* b */ EXPLAIN SELECT 1", "EXPLAIN FORMATTED /* a /* b */ EXPLAIN SELECT 1"},diff --git a/frontend/src/apps/explain-visualizer/plan-parsers.test.ts b/frontend/src/apps/explain-visualizer/plan-parsers.test.tsnew file mode 100644index 0000000000..6a25b7ed5a--- /dev/null+++ b/frontend/src/apps/explain-visualizer/plan-parsers.test.ts@@ -0,0 +1,158 @@+import { describe, expect, test } from "vitest";+import { Engine } from "@/types/proto-es/v1/common_pb";+import {+ VISUALIZER_EXPLAIN_FORMATS,+ type VisualizerEngine,+} from "@/utils/queryPlanEngines";+import { flattenPlan, type PlanTree } from "./plan-model";+import { parsePlan } from "./plan-parsers";++/**+ * What every parser owes its caller, whichever engine it reads.+ *+ * Each engine's own test covers what its plans mean. This one covers what they+ * have in common: that a parser answers rather than throws, and that the tree+ * it returns upholds the model's contract. The cases run over the registry, so+ * an engine added later is held to both without being named here.+ */++const ENGINES = Object.keys(VISUALIZER_EXPLAIN_FORMATS).map(+ Number+) as VisualizerEngine[];++const engineName = (engine: Engine): string =>+ Engine[engine] ?? `ENGINE_${engine}`;++/** Plans captured from real servers, keyed by the directory they sit in. */+const FIXTURES = import.meta.glob("./test-data/*/*", {+ query: "?raw",+ import: "default",+ eager: true,+}) as Record<string, string>;++const fixturesFor = (engine: VisualizerEngine): [string, string][] => {+ const directory = engineName(engine).toLowerCase();+ return Object.entries(FIXTURES).filter(([path]) =>+ path.startsWith(`./test-data/${directory}/`)+ );+};++/**+ * Input no parser should choke on. A plan reaches these parsers from a server,+ * so the shapes that matter are the ones a server or a proxy in front of it can+ * produce: nothing, an error page, a truncated document, or a well-formed+ * document of the wrong kind.+ */+const HOSTILE_INPUT: [string, string][] = [+ ["empty", ""],+ ["blank", " \t\n "],+ ["prose", "could not connect to server"],+ ["an html error page", "<html><body>502 Bad Gateway</body></html>"],+ ["a truncated json object", '{"Plan": {"Node Type": "Seq'],+ ["a truncated xml document", "<ShowPlanXML><BatchSequence>"],+ ["json null", "null"],+ ["a bare number", "0"],+ ["an empty object", "{}"],+ ["an empty array", "[]"],+ ["numbers past what a double holds", '{"Plan": {"Total Cost": 1e400}}'],+ ["a very long line", "x".repeat(200_000)],+ // A document nested past any engine's plan, which JSON.parse itself may+ // refuse: a parser answers for that too rather than letting it escape.+ ["json nested past the stack", `${"[".repeat(20_000)}${"]".repeat(20_000)}`],+];++/** Asserts the model's own contract on a tree, which every parser builds. */+const expectValidTree = (tree: PlanTree, label: string) => {+ const nodes = flattenPlan(tree.root);+ const ids = new Set(nodes.map((node) => node.id));+ expect(ids.size, `${label}: ids are unique within a tree`).toBe(nodes.length);++ for (const node of nodes) {+ const at = `${label}: node ${node.id}`;+ expect(node.nodeType, `${at} names its operation`).not.toBe("");++ const numbers: [string, number | undefined][] = [+ ["startupCost", node.startupCost],+ ["totalCost", node.totalCost],+ ["selfCost", node.selfCost],+ ["rows", node.rows],+ ["width", node.width],+ ["loops", node.loops],+ ["actualRows", node.actualRows],+ ["actualTime", node.actualTime],+ ["selfTime", node.selfTime],+ ];+ for (const [field, value] of numbers) {+ if (value === undefined) continue;+ expect(+ Number.isFinite(value),+ `${at} has a finite ${field}, not ${value}`+ ).toBe(true);+ }++ // What a node adds on top of its children is a part of what it reports,+ // and never negative: both are what planSelfCost and planSelfTime promise.+ // A share of a number the node never stated is not a smaller number, it is+ // a number with nothing behind it, so neither stands on its own.+ if (node.selfCost !== undefined) {+ expect(+ node.totalCost,+ `${at} states a self cost, so it states a total cost`+ ).not.toBeUndefined();+ expect(+ node.selfCost,+ `${at} has a self cost at or above zero`+ ).toBeGreaterThanOrEqual(0);+ expect(+ node.selfCost,+ `${at} adds no more cost than it reports`+ ).toBeLessThanOrEqual((node.totalCost ?? 0) + 1e-6);+ }+ if (node.selfTime !== undefined) {+ expect(+ node.actualTime,+ `${at} states a self time, so it states an actual time`+ ).not.toBeUndefined();+ expect(+ node.selfTime,+ `${at} has a self time at or above zero`+ ).toBeGreaterThanOrEqual(0);+ expect(+ node.selfTime,+ `${at} adds no more time than it reports`+ ).toBeLessThanOrEqual((node.actualTime ?? 0) + 1e-6);+ }+ }+};++describe.each(ENGINES.map((engine) => [engineName(engine), engine] as const))(+ "parsePlan %s",+ (name, engine) => {+ test.each(HOSTILE_INPUT)("answers for %s", (_label, source) => {+ const result = parsePlan(engine, source);+ if (result.ok) {+ // Reading a plan out of such input is a surprise, not a fault: a text+ // plan can be a single line. The tree still has to hold up.+ expectValidTree(result.tree, name);+ return;+ }+ // A viewer that cannot draw says why, in words a reader can act on+ // rather than an empty box.+ expect(result.message.trim()).not.toBe("");+ });++ // A directory holds the plans an engine answers with, which includes the+ // ones it refuses to draw: Hive states no CBO plan where its cost-based+ // optimizer did not plan the statement. Either way the parser answers.+ test("upholds the model on every plan captured from a real server", () => {+ for (const [path, source] of fixturesFor(engine)) {+ const result = parsePlan(engine, source);+ if (!result.ok) {+ expect(result.message.trim(), path).not.toBe("");+ continue;+ }+ expectValidTree(result.tree, path);+ }+ });+ }+);