Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(plsql): trailing ';' leak with hidden tokens (BYT-9367) (#20230)

zchpeter6c950a0f2f3d
diff --git a/backend/plugin/parser/plsql/plsql.go b/backend/plugin/parser/plsql/plsql.go
index b5d40b0ce7..0cc11ce1d2 100644
--- a/backend/plugin/parser/plsql/plsql.go
+++ b/backend/plugin/parser/plsql/plsql.go
@@ -136,7 +136,7 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
// Formula: first token's line - 1 (convert to 0-based) - number of newlines in leading content
stmtBaseLine = startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")
- prevStopTokenIndex = stopToken.GetTokenIndex()
+ prevStopTokenIndex = consumeTrailingSemicolon(tokens.GetAllTokens(), stopToken.GetTokenIndex())
// Skip empty statements
if strings.TrimSpace(stmtText) == "" || stmtText == ";" {
diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.go
index 9c3378a3a4..58a0a9ef0a 100644
--- a/backend/plugin/parser/plsql/split.go
+++ b/backend/plugin/parser/plsql/split.go
@@ -13,8 +13,27 @@ func init() {
base.RegisterSplitterFunc(storepb.Engine_ORACLE, SplitSQL)
}
+// consumeTrailingSemicolon walks forward from stopIdx through hidden-channel
+// tokens (whitespace, comments) looking for a trailing ';' that belongs to the
+// statement ending at stopIdx. Returns the index of the ';' if found before any
+// default-channel token, otherwise stopIdx (no consumption). Used by both
+// SplitSQL and ParsePLSQL to keep the two iterations' separator handling in
+// lockstep — divergence between them is what produced BYT-9367's secondary
+// AST-classification leak.
+func consumeTrailingSemicolon(allTokens []antlr.Token, stopIdx int) int {
+ for nextIdx := stopIdx + 1; nextIdx < len(allTokens); nextIdx++ {
+ next := allTokens[nextIdx]
+ if next.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ return nextIdx
+ }
+ if next.GetChannel() == antlr.TokenDefaultChannel {
+ return stopIdx
+ }
+ }
+ return stopIdx
+}
+
// SplitSQL splits the given SQL statement into multiple SQL statements.
-// TODO(zp): Consolidate with split logic in ParsePLSQL?
func SplitSQL(statement string) ([]base.Statement, error) {
tree, stream, err := ParsePLSQLForStringsManipulation(statement)
if err != nil {
@@ -102,14 +121,15 @@ func SplitSQL(statement string) ([]base.Statement, error) {
},
})
byteOffsetStart = byteOffsetEnd
- // Set prevStopTokenIndex to the last token we want to "consume" for this statement.
- // For statements where the semicolon is a separator (not part of the statement parse tree),
- // we need to skip past the semicolon so it's not included in the next statement's leadingContent.
- prevStopTokenIndex = stmt.GetStop().GetTokenIndex()
- if nextIdx := prevStopTokenIndex + 1; nextIdx < len(tokens.GetAllTokens()) {
- if nextToken := tokens.Get(nextIdx); nextToken.GetTokenType() == parser.PlSqlParserSEMICOLON {
- prevStopTokenIndex = nextIdx
- }
+ // If a trailing ';' was consumed across hidden tokens, advance
+ // byteOffsetStart by the BYTE length of the consumed span — len()
+ // is bytes; ANTLR token indices are runes, so multi-byte UTF-8 in
+ // hidden tokens (e.g., a non-ASCII comment) would diverge.
+ stopIdx := stmt.GetStop().GetTokenIndex()
+ allTokens := tokens.GetAllTokens()
+ prevStopTokenIndex = consumeTrailingSemicolon(allTokens, stopIdx)
+ if prevStopTokenIndex > stopIdx {
+ byteOffsetStart += len(tokens.GetTextFromTokens(allTokens[stopIdx+1], allTokens[prevStopTokenIndex]))
}
}
}
diff --git a/backend/plugin/parser/plsql/test-data/test_split.yaml b/backend/plugin/parser/plsql/test-data/test_split.yaml
index 1da0576583..4c8fb73d3f 100644
--- a/backend/plugin/parser/plsql/test-data/test_split.yaml
+++ b/backend/plugin/parser/plsql/test-data/test_split.yaml
@@ -17,13 +17,13 @@
baseline: 0
start:
line: 1
- column: 17
+ column: 18
end:
line: 1
column: 35
range:
- start: 16
- end: 33
+ start: 17
+ end: 34
empty: false
- description: multiple statements with newlines
input: "\n\t\t\t\tselect * from t;\n\t\t\t\tcreate table table$1 (id int)\n\t\t\t"
@@ -44,13 +44,13 @@
baseline: 1
start:
line: 2
- column: 20
+ column: 21
end:
line: 3
column: 34
range:
- start: 20
- end: 54
+ start: 21
+ end: 55
empty: false
- description: procedure with forward slash separator
input: |-
@@ -139,13 +139,13 @@
baseline: 1
start:
line: 2
- column: 1
+ column: 2
end:
line: 3
column: 17
range:
- start: 18
- end: 35
+ start: 19
+ end: 36
empty: false
- description: anonymous block with forward slash
input: |-
@@ -358,11 +358,134 @@
baseline: 0
start:
line: 1
- column: 19
+ column: 20
end:
line: 3
column: 21
range:
- start: 18
- end: 40
+ start: 19
+ end: 41
+ empty: false
+- description: 'BYT-9367: trailing semicolon with leading space does not leak (cell b)'
+ input: "insert into t values('a',1) ;\n\ninsert into t values('b',2) ;"
+ result:
+ - text: "insert into t values('a',1)"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 28
+ range:
+ start: 0
+ end: 27
+ empty: false
+ - text: "\n\ninsert into t values('b',2)"
+ baseline: 0
+ start:
+ line: 1
+ column: 30
+ end:
+ line: 3
+ column: 28
+ range:
+ start: 29
+ end: 58
+ empty: false
+- description: 'BYT-9367: trailing semicolon with leading inline comment does not leak (cell c)'
+ input: "insert into t values('a',1) /* note */ ;\ninsert into t values('b',2) ;"
+ result:
+ - text: "insert into t values('a',1)"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 28
+ range:
+ start: 0
+ end: 27
+ empty: false
+ - text: "\ninsert into t values('b',2)"
+ baseline: 0
+ start:
+ line: 1
+ column: 41
+ end:
+ line: 2
+ column: 28
+ range:
+ start: 40
+ end: 68
+ empty: false
+- description: 'BYT-9367: trailing semicolon with multiple newlines does not leak (cell d)'
+ input: "insert into t values('a',1)\n\n;\n\ninsert into t values('b',2);"
+ result:
+ - text: "insert into t values('a',1)"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 28
+ range:
+ start: 0
+ end: 27
+ empty: false
+ - text: "\n\ninsert into t values('b',2)"
+ baseline: 2
+ start:
+ line: 3
+ column: 2
+ end:
+ line: 5
+ column: 28
+ range:
+ start: 30
+ end: 59
+ empty: false
+- description: 'BYT-9367: anonymous block followed by SELECT with no separator (cell f, bail-on-default-channel)'
+ input: "BEGIN NULL; END;\nSELECT 1 FROM dual"
+ result:
+ - text: "BEGIN NULL; END;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
+ empty: false
+ - text: "\nSELECT 1 FROM dual"
+ baseline: 0
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 2
+ column: 19
+ range:
+ start: 16
+ end: 35
+ empty: false
+- description: 'BYT-9367: trailing semicolon with leading space at EOF (cell h)'
+ input: "insert into t values('a',1) ;"
+ result:
+ - text: "insert into t values('a',1)"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 28
+ range:
+ start: 0
+ end: 27
empty: false
diff --git a/backend/plugin/parser/plsql/test-data/test_statement_type.yaml b/backend/plugin/parser/plsql/test-data/test_statement_type.yaml
index 9335b71acd..122362db70 100644
--- a/backend/plugin/parser/plsql/test-data/test_statement_type.yaml
+++ b/backend/plugin/parser/plsql/test-data/test_statement_type.yaml
@@ -43,3 +43,12 @@
WHERE FirstName = 'John' AND LastName = 'Doe';
want:
- DELETE
+- statement: "insert into t values('a',1) ;\n\ninsert into t values('b',2) ;"
+ want:
+ - INSERT
+- statement: "insert into t values('a',1) /* note */ ;\ninsert into t values('b',2) ;"
+ want:
+ - INSERT
+- statement: "insert into t values('a',1)\n\n;\n\ninsert into t values('b',2);"
+ want:
+ - INSERT
diff --git a/docs/superpowers/plans/2026-05-07-oracle-splitter-trailing-semicolon-leak.md b/docs/superpowers/plans/2026-05-07-oracle-splitter-trailing-semicolon-leak.md
new file mode 100644
index 0000000000..cf774d6fed
--- /dev/null
+++ b/docs/superpowers/plans/2026-05-07-oracle-splitter-trailing-semicolon-leak.md
@@ -0,0 +1,272 @@
+# Oracle splitter trailing `;` leak (BYT-9367) — implementation plan
+
+> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Fix `plsql.SplitSQL` so a trailing `;` separated from the statement's parse-tree stop token by hidden-channel tokens (whitespace, comments) is correctly consumed by the current statement instead of leaking into the next statement's text/range. Also fix the latent `byteOffsetStart` misalignment surfaced by the Range invariant in the design.
+
+**Architecture:** Replace the single-token SEMI lookahead with a channel-aware loop. After the loop, advance `byteOffsetStart` by `len(GetTextFromTokens(...))` of any tokens the loop consumed (matching the byte-length convention at `split.go:82`), so `Range.Start` of the next statement aligns with where its leadingContent actually begins in source. Note: ANTLR token Start/Stop indices are *rune* indices into a `[]rune` input stream, NOT byte offsets — so the bookkeeping must use `len()` of the materialized text, not stop-index arithmetic.
+
+**Tech Stack:** Go, ANTLR4 (`github.com/antlr4-go/antlr/v4`), `github.com/bytebase/parser/plsql`, YAML test runner (`backend/plugin/parser/base/split_test_runner.go`).
+
+**Spec:** `docs/superpowers/specs/2026-05-07-oracle-splitter-trailing-semicolon-leak-design.md`
+
+**Scope note:** Trino was investigated and ruled out — `trino/parser/TrinoParser.g4:38-40` requires `;` in the `singleStatement` parse tree, so the BYT-9367 leak shape can't occur there. See spec §7 audit.
+
+---
+
+## File Structure
+
+| File | Role | Change |
+|---|---|---|
+| `backend/plugin/parser/plsql/split.go` | Oracle splitter | Replace `split.go:108-113` lookahead block with channel-aware loop + 2-line bookkeeping update |
+| `backend/plugin/parser/plsql/test-data/test_split.yaml` | YAML fixtures | Append 5 new fixtures (cells b, c, d, f, h); 4 existing fixtures will re-record with 1-byte/1-column Range/Start.Column shifts |
+
+## Test plan recap (from spec §6)
+
+Five new fixtures (cells b, c, d, f, h). The current YAML has **15 existing fixtures**; **4 re-record** with 1-byte/1-column Range/Start.Column shifts (`multiple SELECT statements`, `multiple statements with newlines`, `SELECT statements separated by forward slash`, `position semantic: multi-statement with leading whitespace`); **11 stay unchanged** (see spec §6.2 for the full list and rationale per fixture). Text never changes.
+
+---
+
+## Chunk 1: implementation
+
+### Task 1: Establish green baseline
+
+- [ ] **Step 1: Run the existing plsql test suite**
+
+```bash
+go test -count=1 ./backend/plugin/parser/plsql/
+```
+
+Expected: all existing subtests pass. The Edit tool in Task 3 will fail loudly if the source has drifted from the spec's `split.go:108-113` reference, so no separate sed-check is needed.
+
+### Task 2: Add new test fixtures with placeholder results
+
+**Files:**
+- Modify: `backend/plugin/parser/plsql/test-data/test_split.yaml` (append five new cases)
+
+- [ ] **Step 1: Append five plsql test cases**
+
+Append to the END of `backend/plugin/parser/plsql/test-data/test_split.yaml`:
+
+```yaml
+- description: 'BYT-9367: trailing semicolon with leading space does not leak (cell b)'
+ input: "insert into t values('a',1) ;\n\ninsert into t values('b',2) ;"
+ result: []
+- description: 'BYT-9367: trailing semicolon with leading inline comment does not leak (cell c)'
+ input: "insert into t values('a',1) /* note */ ;\ninsert into t values('b',2) ;"
+ result: []
+- description: 'BYT-9367: trailing semicolon with multiple newlines does not leak (cell d)'
+ input: "insert into t values('a',1)\n\n;\n\ninsert into t values('b',2);"
+ result: []
+- description: 'BYT-9367: anonymous block followed by SELECT with no separator (cell f, bail-on-default-channel)'
+ input: "BEGIN NULL; END;\nSELECT 1 FROM dual"
+ result: []
+- description: 'BYT-9367: trailing semicolon with leading space at EOF (cell h)'
+ input: "insert into t values('a',1) ;"
+ result: []
+```
+
+`result: []` is a placeholder — Step 4 below records the expected values via `-record`. Hand-computing positions is error-prone; the `-record` mode is the load-bearing tool.
+
+### Task 3: Apply the fix
+
+**Files:**
+- Modify: `backend/plugin/parser/plsql/split.go:108-113`
+
+- [ ] **Step 1: Apply the plsql fix**
+
+In `backend/plugin/parser/plsql/split.go`, replace lines 108-113:
+
+```go
+ prevStopTokenIndex = stmt.GetStop().GetTokenIndex()
+ if nextIdx := prevStopTokenIndex + 1; nextIdx < len(tokens.GetAllTokens()) {
+ if nextToken := tokens.Get(nextIdx); nextToken.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ prevStopTokenIndex = nextIdx
+ }
+ }
+```
+
+with:
+
+```go
+ // Walk forward through hidden-channel tokens (whitespace, comments) to
+ // find a trailing ';' belonging to this statement. Bail on the first
+ // default-channel non-';' token — that's the start of the next statement.
+ loopStart := stmt.GetStop().GetTokenIndex()
+ prevStopTokenIndex = loopStart
+ allTokens := tokens.GetAllTokens()
+ for nextIdx := prevStopTokenIndex + 1; nextIdx < len(allTokens); nextIdx++ {
+ next := allTokens[nextIdx]
+ if next.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ prevStopTokenIndex = nextIdx
+ break
+ }
+ if next.GetChannel() == antlr.TokenDefaultChannel {
+ break
+ }
+ }
+ // If the loop consumed any tokens, advance byteOffsetStart by the
+ // byte length of those consumed tokens so the next statement's
+ // Range.Start lands at the byte AFTER the consumed ';' (matching
+ // where its leadingContent actually begins in source).
+ //
+ // IMPORTANT: use len(GetTextFromTokens(...)) — Go's len() on a
+ // string is byte length, while ANTLR token Start/Stop indices are
+ // rune indices into the input stream's []rune. For ASCII the
+ // difference is zero, but multi-byte UTF-8 inside hidden tokens
+ // (e.g., a comment containing non-ASCII characters) would diverge.
+ // This matches the byte-offset arithmetic at line 82.
+ if prevStopTokenIndex > loopStart {
+ byteOffsetStart += len(tokens.GetTextFromTokens(allTokens[loopStart+1], allTokens[prevStopTokenIndex]))
+ }
+```
+
+The `antlr` import is already present at `split.go:4`. The `parser.PlSqlParserSEMICOLON` symbol is unchanged.
+
+- [ ] **Step 2: Format**
+
+```bash
+gofmt -w backend/plugin/parser/plsql/split.go
+```
+
+### Task 4: Record YAML and verify text/range invariants
+
+**Files:**
+- Read/diff: `backend/plugin/parser/plsql/test-data/test_split.yaml`
+
+- [ ] **Step 1: Record**
+
+```bash
+go test -count=1 ./backend/plugin/parser/plsql/ -run TestPLSQLSplitSQL -args -record
+```
+
+This populates `result:` for the new cases and re-records all others.
+
+- [ ] **Step 2: Diff and verify text invariants for new cells**
+
+```bash
+git diff backend/plugin/parser/plsql/test-data/test_split.yaml
+```
+
+For each of the five NEW cases, verify stmt 2's `text` is CLEAN:
+
+| cell | input shape | expected stmt 2 text invariant |
+|---|---|---|
+| b | `,1) ;\n\nstmt2 ;` | starts with `"\n\n"`, NO leading `;`, NO leading space |
+| c | `,1) /* note */ ;\nstmt2 ;` | starts with `"\n"`, NO leading `;`, NO leading comment |
+| d | `,1)\n\n;\n\nstmt2;` | exactly 2 unit_statements (the standalone `;` matches sql_script's bare-`SEMICOLON` alternative, not a unit_statement, so the splitter — which only iterates `IUnit_statementContext` — sees 2 children). Stmt 2 text starts with pure whitespace, NO leading `;` |
+| f | `BEGIN NULL; END;\nSELECT 1 FROM dual` | 2 stmts: stmt 1 text = `"BEGIN NULL; END;"`; stmt 2 text = `"\nSELECT 1 FROM dual"` (single `\n` leading, NO `;` leak) |
+| h | `,1) ;` (EOF, no stmt 2) | 1 stmt only, text = `"insert into t values('a',1)"`, NO `;` in text |
+
+If any invariant is violated, the fix is wrong — investigate before continuing.
+
+- [ ] **Step 3: Diff and verify expected existing-fixture changes**
+
+The same diff should show 1-byte/1-column shifts (NOT text changes) in exactly 4 of the 15 existing fixtures:
+
+| fixture (line) | expected change |
+|-------------------------------------------------------------------------|------------------------------------------------------|
+| `multiple SELECT statements` (line 1) | stmt 2: `range.start` 16→17, `range.end` 33→34, `start.column` 17→18 |
+| `multiple statements with newlines` (line 28) | stmt 2: `range.start` 20→21, `range.end` 54→55, `start.column` 20→21 |
+| `SELECT statements separated by forward slash` (line 120) | stmt 2 (after `/`): `range.start` 18→19, `range.end` 35→36, `start.column` 1→2 |
+| `position semantic: multi-statement with leading whitespace` (line 342) | stmt 2: `range.start` 18→19, `range.end` 40→41, `start.column` 19→20 |
+
+For each: verify `text` is UNCHANGED. Any text change is a flag.
+
+For the OTHER 11 existing fixtures (lines 55, 79, 150 — needSemicolon-with-`/`; lines 172, 205, 220, 235, 238, 241, 312, 327 — single-statement / parse-error / single-stmt-with-leading-whitespace), verify ZERO changes — they don't trigger the bookkeeping update.
+
+- [ ] **Step 4: Run the test suite in normal (non-record) mode**
+
+```bash
+go test -count=1 -v ./backend/plugin/parser/plsql/ -run TestPLSQLSplitSQL
+```
+
+Expected: all subtests pass, including the five new BYT-9367 subtests.
+
+### Task 5: Lint and build
+
+- [ ] **Step 1: Lint**
+
+```bash
+golangci-lint run --allow-parallel-runners ./backend/plugin/parser/plsql/...
+```
+
+Run repeatedly until no issues remain. Use `--fix` to auto-fix:
+
+```bash
+golangci-lint run --fix --allow-parallel-runners ./backend/plugin/parser/plsql/...
+```
+
+Expected: clean.
+
+- [ ] **Step 2: Build the full backend**
+
+```bash
+go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go
+```
+
+Expected: builds successfully.
+
+### Task 6: Commit
+
+- [ ] **Step 1: Stage source and test changes**
+
+```bash
+git add backend/plugin/parser/plsql/split.go \
+ backend/plugin/parser/plsql/test-data/test_split.yaml
+```
+
+- [ ] **Step 2: Commit**
+
+```bash
+git commit -m "$(cat <<'EOF'
+fix(plsql): trailing ';' leak when whitespace/comment separates stop and ';'
+
+BYT-9367. The plsql splitter's "advance past trailing ';'" lookahead
+inspected only the token immediately after the parse-tree stop, so any
+hidden-channel token (whitespace, comment) between the stop and the ';'
+prevented consumption. The ';' then leaked into the next statement's
+text. For Oracle this surfaced as ORA-00900 at position 1.
+
+Replace the single-token lookahead with a channel-aware loop that skips
+hidden-channel tokens until it finds ';' (consume) or a default-channel
+non-';' token (bail). After the loop, advance byteOffsetStart by the byte
+length of any consumed tokens so the next statement's Range.Start aligns
+with where its leadingContent actually begins in source. This also
+quietly fixes a pre-existing 1-byte off-by-one in the immediate-';' case
+(visible in 4 existing fixtures).
+
+Tests:
+- 5 new YAML fixtures (cells b/c/d/f/h of the design gap).
+- 4 existing fixtures re-recorded with 1-byte/1-column Range/Start.Column
+ shifts (text unchanged): multiple SELECT statements, multiple statements
+ with newlines, SELECT statements separated by forward slash, position
+ semantic: multi-statement with leading whitespace.
+
+Trino was investigated and ruled out — TrinoParser.g4 requires ';' in
+the singleStatement parse tree, so the leak shape can't occur there.
+
+Spec: docs/superpowers/specs/2026-05-07-oracle-splitter-trailing-semicolon-leak-design.md
+
+Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
+EOF
+)"
+```
+
+- [ ] **Step 3: Verify clean status**
+
+```bash
+git status
+```
+
+Expected: working tree clean (besides the spec/plan docs from earlier commits).
+
+---
+
+## Out of scope (for reference)
+
+- **Trino splitter.** Investigated; not affected. `TrinoParser.g4` requires `;` in `singleStatement`.
+- **Multi-`;` runs (`stmt;;stmt2`).** Pre-existing limitation; not made worse.
+- **Driver-level integration tests.** No existing testcontainer infra targeting this code path.
+- **Defensive `leadingContent` rebuild.** Long-term hardening idea; channel-aware loop + bookkeeping closes the documented gap.
+- **Variable rename of `prevStopTokenIndex`.** Mildly confusing in the inner loop's local context but renaming would balloon the diff.
diff --git a/docs/superpowers/specs/2026-05-07-oracle-splitter-trailing-semicolon-leak-design.md b/docs/superpowers/specs/2026-05-07-oracle-splitter-trailing-semicolon-leak-design.md
new file mode 100644
index 0000000000..2545c30ec3
--- /dev/null
+++ b/docs/superpowers/specs/2026-05-07-oracle-splitter-trailing-semicolon-leak-design.md
@@ -0,0 +1,300 @@
+# Oracle splitter trailing `;` leak (BYT-9367) — design
+
+## 1. Problem
+
+Two simple Oracle `INSERT`s submitted as one Bytebase issue:
+
+```sql
+insert into ODISTG.ODS_DATA_RETENTION_DROP_PAR_CONFIG values('RPT', 'DBA_CHANGE_INFO ',183) ;
+
+insert into ODISTG.ODS_DATA_RETENTION_DROP_PAR_CONFIG values('RPT', 'CHANGE_INFO_SERVICE ',183) ;
+```
+
+Issue execution emits `Command Execute (2)`: the first `INSERT` succeeds, the
+second fails with `ORA-00900: invalid SQL statement error occur at position: 1`.
+SQL Editor on superficially the same input succeeds; the difference is paste-
+normalization, not a separate code path (see §3).
+
+Both layers go through the same splitter, so the bug must live there. It does.
+
+## 2. Root cause
+
+`backend/plugin/parser/plsql/split.go:108-113` — the "advance past trailing `;`
+so it does not bleed into the next statement's `leadingContent`" step:
+
+```go
+prevStopTokenIndex = stmt.GetStop().GetTokenIndex()
+if nextIdx := prevStopTokenIndex + 1; nextIdx < len(tokens.GetAllTokens()) {
+ if nextToken := tokens.Get(nextIdx); nextToken.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ prevStopTokenIndex = nextIdx
+ }
+}
+```
+
+For an `INSERT`, `stmt.GetStop()` is the closing `)`. ANTLR's
+`CommonTokenStream` keeps whitespace and comments on the **hidden channel** but
+still in the stream. Around the first `;` the tokens are:
+
+```
+… 16: ')' (channel 0)
+ 17: ' ' (channel 1, hidden)
+ 18: ';' (channel 0)
+ 19: '\n' (channel 1, hidden)
+ 20: '\n' (channel 1, hidden)
+ 21: 'insert' (channel 0)
+…
+```
+
+`tokens.Get(stopIdx + 1)` returns the whitespace token, not `;`. The equality
+check fails, `prevStopTokenIndex` stays on `)`, and the next statement's
+`leadingContent` (computed by `tokens.GetTextFromTokens(prevStop+1,
+nextStart-1)`) absorbs the space, the `;`, and both newlines. Statement #2 goes
+to Oracle as `" ;\n\ninsert into …"` — Oracle parses position 1 and reports
+`ORA-00900`.
+
+Trigger-condition table:
+
+| input ending | second stmt leading | result |
+|-----------------------|---------------------|---------------------------|
+| `,183) ;` (space) | `" ;\n…"` | **ORA-00900 (broken)** |
+| `,183);` (no space) | `"\n…"` | OK |
+| `,183)` (no `;`) | `"\n…"` | OK |
+
+Gated on whitespace (or any hidden-channel token) between the statement's stop
+token and the trailing `;`.
+
+The bug is gated on the grammar shape "`;` is at sql_script level (outside
+`unit_statement`), not inside the rule itself." Per `PlSqlParser.g4:32-37`,
+ordinary DML/DDL is wrapped by `sql_script: ((sql_plus_command |
+unit_statement) SEMICOLON? | SEMICOLON)* EOF`, so `stmt.GetStop()` of a SELECT
+or INSERT is the last meaningful token (e.g., `t1`, `)`), not the trailing
+`;`. `needSemicolon` rules (anonymous blocks, procedure/function/package/
+trigger bodies) include their `;` in the rule itself
+(`anonymous_block: ... END SEMICOLON`, `create_function_body: ... SEMICOLON`),
+so `stmt.GetStop()` IS the `;` for those — the loop starts past it and does
+not (in the absence of a stray double-`;`) advance further.
+
+## 3. Why "SQL Editor worked"
+
+There is one Oracle splitter (`backend/plugin/parser/plsql/split.go:13`,
+`base.RegisterSplitterFunc(storepb.Engine_ORACLE, SplitSQL)`). All callers route
+through it:
+
+| caller | path |
+|-----------------------|--------------------------------------------------|
+| Issue / rollout | `Driver.Execute` → `plsqlparser.SplitSQL` |
+| SQL Editor (Admin) | `Driver.QueryConn` → `plsqlparser.SplitSQL` |
+| SQL Editor (Query) | `parserbase.SplitMultiSQL` → `QueryConn` → same |
+
+So byte-identical input must hit the same leak. The SQL Editor screenshot
+"working" is best explained by paste/format normalization stripping the space
+before `;` (which §2 shows avoids the leak). No path-specific bypass exists.
+
+## 4. Design gap
+
+The splitter has two linked invariants that the original 1-token lookahead
+encoded only partially:
+
+**Invariant A — Text correctness.** After processing statement N,
+`prevStopTokenIndex` must point to the last token that "belongs" to N
+(including any trailing `;` separator), so statement N+1's `leadingContent`
+starts cleanly at the first non-belonging token. The OLD code's lookahead
+encoded a weaker assumption: that `;`, when present, sits **immediately** after
+`stmt.GetStop()`. That holds for tightly written SQL (`stmt;`) and breaks for
+any statement with hidden-channel tokens before `;`.
+
+**Invariant B — Range alignment.** `byteOffsetStart` of statement N+1 (which
+becomes `Range.Start` of N+1 and the input to `CalculateLineAndColumn` for
+`Start.Line`/`Start.Column`) must equal the byte position of the first
+character of N+1's `leadingContent`, so that `source[Range.Start:Range.End]`
+reproduces `Text` and `Start.Line`/`Column` points at where the statement
+actually begins in source. The OLD code never advanced `byteOffsetStart`
+after the lookahead, so existing fixtures already drift by 1 byte for the
+immediate-`;` case (see §6.2 row 1). BYT-9367 inputs (where the loop now
+consumes whitespace + `;`) would amplify this to N bytes, visibly drifting
+line/column for downstream consumers (e.g., BYT-9089 rollout error mapping).
+
+The corrected statement of intent: **(A) scan forward through hidden-channel
+tokens to find the trailing `;`, if any; consume it and stop. Otherwise (the
+next default-channel token is not `;`), do not advance. (B) If the scan
+consumed any tokens, advance `byteOffsetStart` (= next statement's
+`Range.Start`) by the byte length of those consumed tokens, so it lands at
+the first byte of the next statement's `leadingContent`.**
+
+## 5. Fix
+
+Replace the single-step lookahead at `split.go:108-113` with a loop plus a
+post-loop bookkeeping update:
+
+```go
+loopStart := stmt.GetStop().GetTokenIndex()
+prevStopTokenIndex = loopStart
+allTokens := tokens.GetAllTokens()
+for nextIdx := prevStopTokenIndex + 1; nextIdx < len(allTokens); nextIdx++ {
+ next := allTokens[nextIdx]
+ if next.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ prevStopTokenIndex = nextIdx
+ break
+ }
+ if next.GetChannel() == antlr.TokenDefaultChannel {
+ // Hit a real next-statement token; this stmt has no trailing ';'.
+ break
+ }
+}
+// If the loop consumed any tokens, advance byteOffsetStart by the byte length
+// of those consumed tokens so the next statement's Range.Start lands at the
+// byte AFTER the consumed `;` (matching where its leadingContent actually
+// begins in source). Use len(GetTextFromTokens(...)) — Go's len() on a
+// string is byte length, while ANTLR token Start/Stop indices are *rune*
+// indices into the input stream (input_stream.go: data []rune). For ASCII
+// the difference is zero, but multi-byte UTF-8 inside hidden tokens (e.g.,
+// a comment containing non-ASCII characters) would diverge. Match the
+// byte-offset arithmetic at line 82 by using string length.
+if prevStopTokenIndex > loopStart {
+ byteOffsetStart += len(tokens.GetTextFromTokens(allTokens[loopStart+1], allTokens[prevStopTokenIndex]))
+}
+```
+
+Properties:
+
+* **Grammar-stable.** The skip predicate is `channel != DEFAULT`. We do not
+ enumerate token types. `PlSqlLexer.g4` routes whitespace (`SPACES`,
+ line 2638) and all comment forms (`SINGLE_LINE_COMMENT`,
+ `MULTI_LINE_COMMENT`, `REMARK_COMMENT`, lines 2621-2624) to `channel(HIDDEN)`.
+ If the grammar adds another hidden-channel token type later, no code change
+ is needed.
+* **Per-fixture behavior** is enumerated in §6 (cells a–h, plus existing
+ fixture re-records).
+* **Bounded.** The loop bails on the first default-channel token that isn't
+ `;`, so it cannot consume a `;` that belongs to a later statement.
+* **UTF-8 safe.** Bookkeeping uses `len(GetTextFromTokens(...))` (byte
+ length) rather than `Stop()-Stop()` rune-index arithmetic, matching the
+ existing byte-offset convention at `split.go:82`.
+* **Local.** No signature change, no new helper, no callsite churn.
+
+## 6. Test plan
+
+### 6.1 Splitter unit tests — new fixtures
+
+Add cases to `backend/plugin/parser/plsql/test-data/test_split.yaml`. Each
+covers a distinct cell of the design gap:
+
+| # | cell | input fixture | new? |
+|---|-------------------------------------------------------|-----------------------------------------------------------------------|------|
+| a | immediate `;` (non-needSemicolon) | (existing) `multiple SELECT statements` | no, will re-record (§6.2) |
+| b | whitespace before `;` (BYT-9367 exact) | `insert into t values('a',1) ;\n\ninsert into t values('b',2) ;` | yes |
+| c | comment before `;` | `insert into t values('a',1) /* note */ ;\ninsert into t values('b',2) ;` | yes |
+| d | multi-newline / mixed whitespace before `;` | `insert into t values('a',1)\n\n;\n\ninsert into t values('b',2);` | yes |
+| e | no `;` at end of input | (existing) `multiple statements with newlines` | no, will re-record (§6.2) |
+| f | hidden tokens then default-channel non-`;` (no sep.) | `BEGIN NULL; END;\nSELECT 1 FROM dual` (anonymous block then SELECT, no `;` between) | yes |
+| g | `needSemicolon` branch (anonymous block, `/`) | (existing) `anonymous block with forward slash` et al. | no, unchanged |
+| h | trailing `;` with hidden tokens before it at EOF | `insert into t values('a',1) ;` (no second statement) | yes |
+
+**Five new cases.** Cell (f) input is `BEGIN NULL; END;\nSELECT 1 FROM dual`:
+the anonymous block (needSemicolon, stop = `;`) is followed by a SELECT with
+only `\n` between them. The loop starts at `;+1`, walks past `\n` (HIDDEN),
+hits `SELECT` (DEFAULT, not SEMI) → bails. `prevStopTokenIndex` unchanged,
+bookkeeping does not fire. This locks the bail-on-default-channel branch
+with a non-`/` token (the existing forward-slash fixtures bail on
+`FORWARD_SLASH`; cell (f) bails on a regular keyword). Cell (h) covers the
+EOF bound of the loop and the bookkeeping firing without a downstream
+consumer.
+
+For non-`needSemicolon` statements (`INSERT`/`UPDATE`/`DELETE`/`SELECT`/DDL),
+the trailing `;` is excluded from `Statement.Text` (existing behavior at
+`split.go:71-77`). For `needSemicolon` statements (anonymous blocks, procedure
+bodies, etc.), the trailing `;` is preserved.
+
+### 6.2 Existing fixtures — Range value changes
+
+The current `test_split.yaml` has **15 existing fixtures**. Of these, **4
+will re-record** (1-byte/1-column Range/Start.Column shifts) and **11 will
+remain unchanged**.
+
+The 4 that re-record exercise the "non-needSemicolon stmt with immediate
+`;` consumed by lookahead" path that the new bookkeeping now correctly
+accounts for:
+
+| fixture (line in `test_split.yaml`) | what shifts |
+|-------------------------------------------------------------------------|----------------------------------------------|
+| `multiple SELECT statements` (line 1) | stmt 2 `Range.Start` 16→17; `Range.End` 33→34; `Start.Column` 17→18 |
+| `multiple statements with newlines` (line 28) | stmt 2: `Range.Start` 20→21; `Range.End` 54→55; `Start.Column` 20→21 |
+| `SELECT statements separated by forward slash` (line 120) | stmt 2 (after `/`): `Range.Start` 18→19; `Range.End` 35→36; `Start.Column` 1→2 |
+| `position semantic: multi-statement with leading whitespace` (line 342) | stmt 2: `Range.Start` 18→19; `Range.End` 40→41; `Start.Column` 19→20 |
+
+`Text`, `End.Line`/`End.Column`, and `Empty` are unchanged. Re-record via
+`go test -args -record` and verify in diff that ONLY Range/Start.Column
+shift, NEVER Text. Any Text change would indicate a bug elsewhere.
+
+The 11 unchanged fixtures:
+
+| fixture (line) | why unchanged |
+|-------------------------------------------------------------|--------------------------------------------------------------|
+| `procedure with forward slash separator` (55) | needSemicolon, stop = block's own `;`; loop walks `\n` HIDDEN and bails on `/` DEFAULT non-SEMI; no consumption |
+| `two procedures with forward slash separator` (79) | same as above |
+| `anonymous block with forward slash` (150) | same as above |
+| `ALTER TABLE with PARTITION` (172) | single-statement; no next statement to be affected |
+| `CALL procedure` (205) | single-statement |
+| `DROP TABLESPACE with CASCADE CONSTRAINTS` (220) | single-statement |
+| `DROP TABLESPACE CASCADE alone should error` (235) | parse-error path; splitter never reaches lookahead |
+| `DROP TABLESPACE then CASCADE should error` (238) | same as above |
+| `CREATE TABLE with ROW STORE COMPRESS ADVANCED` (241) | single-statement |
+| `position semantic: leading newlines` (312) | single-statement |
+| `position semantic: leading spaces and newlines` (327) | single-statement |
+
+### 6.3 Driver-level integration tests — skipped
+
+There is no existing testcontainer infrastructure for the Oracle driver
+package targeting this code path. Building it for one bug isn't proportional.
+The downstream chain `Statement.Text` → `conn.ExecContext` → go-ora performs
+no transformation, so once the splitter emits clean text the rest is correct
+by construction. §6.1's cell-coverage matrix is the load-bearing surface.
+
+## 7. Audit (other engines / other splitters)
+
+Cross-engine audit:
+
+| splitter | mechanism | status |
+|-----------------------------------------------------|---------------------------------------------------------------|---------------------------------------|
+| `plsql.SplitSQL` | parse-tree walk + `prevStopTokenIndex` + 1-token lookahead | **fixed in this PR** |
+| `plsql.SplitSQLForCompletion` | parse-tree walk, no `leadingContent` computation | not affected |
+| `base.SplitSQLByLexer` (snowflake, …) | token-stream walk, `;` always lands in current buffer | not affected |
+| `mysql.SplitSQL`, `tsql.SplitSQL`, `pg.SplitSQL`, `tidb.SplitSQL` | no `prevStopTokenIndex` / single-token-`;`-lookahead pattern | not affected |
+| `trino.splitByParser` (`trino/split.go:88-91`) | parse-tree walk + 1-token lookahead AT FIRST GLANCE looks similar, BUT: `singleStatement: statement SEMICOLON_` (`TrinoParser.g4:38-40`) **requires** `;` in the parse tree; `singleStmt.GetStop()` is the `;` itself; `rangeEnd = endToken.GetStop() + 1` already extends past `;`. The 1-token lookahead handles a different case: a stray DOUBLE-`;` (empty statement absorption), not the BYT-9367 leak shape. | **not affected** |
+| `trino.splitByTokenizer` fallback | tokenizer-based, different mechanism | not affected (tokenizer in `tokenizer/tokenizer.go:381` walks character-by-character; `;` always terminates a buffer cleanly) |
+
+## 8. Risk and rollback
+
+* **Blast radius.** `plsql.SplitSQL` is shared across Oracle issue execution
+ and SQL Editor (Admin and Query).
+* **Existing test fixture re-records.** Four plsql fixtures (§6.2) will see
+ 1-byte/1-column numerical shifts in Range/Start.Column. No Text changes.
+ The PR description and commit message must call out these specific fixtures
+ and what shifted, so a reviewer can scan the diff and confirm the changes
+ are mechanical re-records, not bugs.
+* **Range correctness improvement.** The bookkeeping fix improves Range
+ alignment for the existing immediate-`;` case (was off by 1 byte;
+ now correct) — this is a quiet *improvement* shipped alongside the
+ BYT-9367 leak fix, for free. Downstream consumers using
+ `source[Range.Start:Range.End]` to reproduce Text will now get matching
+ bytes for these fixtures (previously they got bytes shifted by 1).
+* **Rollback.** Revert the one commit. No data, schema, or proto changes;
+ no new dependencies; no callsite changes.
+
+## 9. Out of scope
+
+* **Multi-`;` runs (`stmt;;stmt2`).** The new loop consumes one `;` per
+ statement. If multiple `;` appear with hidden tokens between, only the
+ first is consumed and subsequent ones leak. Pre-existing limitation, not
+ made worse by this fix; document if reported.
+* **Empty-statement edge case after `needSemicolon` blocks** (e.g.,
+ `BEGIN NULL; END;\n;\nSELECT 1 FROM dual;` where the standalone `;` is a
+ PL/SQL empty-statement boundary). The new loop consumes that standalone
+ `;` rather than emitting an empty stmt, which actually matches existing
+ behavior at `plsql.go:142` (`stmtText == ";"` is skipped). Documented as
+ a known shape; not regressed.
+* **Defensive `leadingContent` rebuild.** Mentioned as long-term hardening
+ in the original Linear analysis. The channel-aware loop + bookkeeping
+ closes the documented design gap.
+* **Full plan→issue→rollout integration test.** Path is exercised by other
+ rollout tests; adds no marginal coverage.