25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
import type { SqlSemanticSpan, SqlSemanticToken } from "@/lib/sql/semantic/types";
const WORD_START = /[A-Za-z_@$#\p{ID_Start}]/u;
const WORD_PART = /[A-Za-z0-9_@$#\p{ID_Continue}]/u;
function characterAt(input: string, index: number): string {
const codePoint = input.codePointAt(index);
return codePoint === undefined ? "" : String.fromCodePoint(codePoint);
}
function token(kind: SqlSemanticToken["kind"], text: string, start: number, end: number, depth: number, quote?: string, closed?: boolean): SqlSemanticToken {
return {
kind,
text,
normalized: kind === "word" ? text.toLowerCase() : text,
span: { start, end },
depth,
quote,
...(closed === undefined ? {} : { closed }),
};
}
function readQuoted(input: string, start: number, open: string, close: string): { end: number; closed: boolean } {
let index = start + open.length;
while (index < input.length) {
if (input.startsWith(close, index)) {
if (input.startsWith(close + close, index)) {
index += close.length * 2;
continue;
}
return { end: index + close.length, closed: true };
}
index += 1;
}
return { end: input.length, closed: false };
}
// Same doubled-quote escaping as readQuoted, plus (when allowBackslashEscape) MySQL-style `\x`
// escaping inside '...' strings. Backslash-escape recognition is NOT safe to apply
// unconditionally: a string that merely *ends* in a literal backslash right before the closing
// quote (e.g. a Windows path literal `'C:\Program Files\'` under Postgres, whose default
// standard_conforming_strings=on gives backslash no special meaning) would have its real closing
// quote misread as escaped, swallowing the rest of the query as a phantom string. So this is
// gated per-dialect by the caller (see tokenizeSqlSemantic's `mysqlBackslashEscape` option) rather
// than applied everywhere.
function readQuotedString(input: string, start: number, quote: string, allowBackslashEscape: boolean): { end: number; closed: boolean } {
let index = start + 1;
while (index < input.length) {
if (allowBackslashEscape && input[index] === "\\" && index + 1 < input.length) {
index += 2;
continue;
}
if (input[index] === quote) {
if (input[index + 1] === quote) {
index += 2;
continue;
}
return { end: index + 1, closed: true };
}
index += 1;
}
return { end: input.length, closed: false };
}
const DOLLAR_QUOTE_TAG_PATTERN = /\$[A-Za-z_0-9]*\$/y;
/**
* Matches a PostgreSQL dollar-quote tag (`$$` or `$tag$`) starting exactly at `index`, without
* allocating a substring: a sticky ("y") regex anchors its match to `lastIndex` and scans the
* original string in place, unlike `pattern.exec(input.slice(index))` which copies everything
* from `index` to the end of `input` on every call.
*/
export function matchDollarQuoteTag(input: string, index: number): string | undefined {
DOLLAR_QUOTE_TAG_PATTERN.lastIndex = index;
return DOLLAR_QUOTE_TAG_PATTERN.exec(input)?.[0];
}
/**
* options.mysqlDashCommentRequiresWhitespace opts into MySQL's rule that a bare "--" only starts
* a line comment when followed by whitespace/EOL, since MySQL reserves unspaced "--" for
* double-negation (e.g. `SELECT 1--1`). Defaults to false so every existing caller keeps today's
* dialect-generic "-- always starts a comment" behavior; only callers targeting a confirmed
* MySQL-family grammar should opt in.
*
* options.mysqlBackslashEscape opts into MySQL-style `\x` escaping inside '...' strings. Also
* dialect-gated (not a global default -- see readQuotedString's doc comment for why an
* unconditional default is unsafe): only dialects confirmed to actually use backslash escaping by
* convention (MySQL and its close wire-protocol/grammar clones) should opt in.
*
* By default "..." is always identifier quoting (never a string literal), matching MySQL's own
* dialect adapter (see semantic/dialect.ts), which lists '"' as one of its valid identifierQuotes
* -- this is correct for every dialect where "..." unconditionally means identifier (Postgres,
* SQL Server's ANSI mode, MySQL running with the ANSI_QUOTES sql_mode) and is what every existing
* caller other than sqlCompletion.ts's literal-masking layer wants.
*
* options.mysqlDoubleQuoteIsString flips the emitted *kind* of a "..." span to "string" instead of
* "quoted_identifier" -- it does not affect how the span itself is scanned (see readQuotedString's
* call below, gated only by mysqlBackslashEscape). Whether ANSI_QUOTES is actually enabled on the
* connected server is runtime state this tokenizer can't see, so callers that want to model MySQL's
* default (non-ANSI_QUOTES) sql_mode can opt in per call site instead of this being a global
* default; leaving it off keeps modeling the ANSI_QUOTES (identifier) interpretation. As of
* sqlCompletion.ts's maskSqlLiteralsAndComments, no caller passes this true anymore -- masking now
* decides string-vs-identifier per token position instead of per dialect (see that function's doc
* comment) -- but the option stays available as a general tokenizer capability.
*/
export function tokenizeSqlSemantic(input: string, dialectId = "mysql", options?: { mysqlDashCommentRequiresWhitespace?: boolean; mysqlBackslashEscape?: boolean; mysqlDoubleQuoteIsString?: boolean; squareBracketsAsDelimiters?: boolean }): SqlSemanticToken[] {
const tokens: SqlSemanticToken[] = [];
const mysqlDashCommentRequiresWhitespace = !!options?.mysqlDashCommentRequiresWhitespace;
const mysqlBackslashEscape = !!options?.mysqlBackslashEscape;
const mysqlDoubleQuoteIsString = !!options?.mysqlDoubleQuoteIsString;
let index = 0;
let depth = 0;
while (index < input.length) {
const start = index;
const ch = characterAt(input, index);
const next = input[index + 1] ?? "";
if (/\s/.test(ch)) {
index += 1;
continue;
}
if (ch === "-" && next === "-" && (!mysqlDashCommentRequiresWhitespace || index + 2 >= input.length || /\s/.test(input[index + 2] ?? ""))) {
index += 2;
while (index < input.length && input[index] !== "\n" && input[index] !== "\r") index += 1;
tokens.push(token("comment", input.slice(start, index), start, index, depth));
continue;
}
if (ch === "#" && (dialectId === "mysql" || dialectId === "doris")) {
index += 1;
while (index < input.length && input[index] !== "\n" && input[index] !== "\r") index += 1;
tokens.push(token("comment", input.slice(start, index), start, index, depth));
continue;
}
if (ch === "#" && dialectId === "postgres") {
index += 1;
tokens.push(token("operator", ch, start, index, depth));
continue;
}
if (ch === "/" && next === "*") {
index += 2;
while (index < input.length && !(input[index] === "*" && input[index + 1] === "/")) index += 1;
index = Math.min(input.length, index + (index < input.length ? 2 : 0));
tokens.push(token("comment", input.slice(start, index), start, index, depth));
continue;
}
const oracleQuoteStart = (ch === "n" || ch === "N") && (next === "q" || next === "Q") ? index + 1 : index;
if (dialectId === "oracle" && (input[oracleQuoteStart] === "q" || input[oracleQuoteStart] === "Q") && input[oracleQuoteStart + 1] === "'" && input[oracleQuoteStart + 2]) {
const opener = characterAt(input, oracleQuoteStart + 2);
const closer = ({ "[": "]", "{": "}", "(": ")", "<": ">" } as Record<string, string>)[opener] ?? opener;
const end = input.indexOf(closer + "'", oracleQuoteStart + 2 + opener.length);
index = end < 0 ? input.length : end + closer.length + 1;
tokens.push(token("string", input.slice(start, index), start, index, depth, oracleQuoteStart === start ? "q'" : "nq'", end >= 0));
continue;
}
if (ch === "'") {
const quoted = readQuotedString(input, start, "'", mysqlBackslashEscape);
index = quoted.end;
tokens.push(token("string", input.slice(start, index), start, index, depth, "'", quoted.closed));
continue;
}
if (ch === "$") {
const marker = matchDollarQuoteTag(input, start);
if (marker) {
const closing = input.indexOf(marker, start + marker.length);
index = closing < 0 ? input.length : closing + marker.length;
tokens.push(token("string", input.slice(start, index), start, index, depth, marker, closing >= 0));
continue;
}
}
if (ch === '"') {
// readQuotedString(...,false) is byte-for-byte identical to readQuoted for this quote pair
// (both use doubled-quote-only escaping), so routing every dialect through readQuotedString
// here is a no-op unless mysqlBackslashEscape is set -- which now matters independently of
// mysqlDoubleQuoteIsString: a MySQL-family "..." kept as a quoted_identifier (see that
// option's doc comment above) can still contain a backslash-escaped quote and must be read
// the same way a "..." string would be, or it desyncs the rest of the token stream.
const quoted = readQuotedString(input, start, '"', mysqlBackslashEscape);
index = quoted.end;
const kind = mysqlDoubleQuoteIsString ? "string" : "quoted_identifier";
tokens.push(token(kind, input.slice(start, index), start, index, depth, '"', quoted.closed));
continue;
}
if (ch === "`") {
const quoted = readQuoted(input, start, "`", "`");
index = quoted.end;
tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, "`", quoted.closed));
continue;
}
if (ch === "[" && !options?.squareBracketsAsDelimiters) {
const quoted = readQuoted(input, start, "[", "]");
index = quoted.end;
tokens.push(token("quoted_identifier", input.slice(start, index), start, index, depth, "[", quoted.closed));
continue;
}
if (ch === ":" || ch === "?") {
index += ch.length;
while (index < input.length) {
const part = characterAt(input, index);
if (!WORD_PART.test(part)) break;
index += part.length;
}
tokens.push(token("parameter", input.slice(start, index), start, index, depth));
continue;
}
if (/[0-9]/.test(ch)) {
index += 1;
while (index < input.length && /[0-9.]/.test(input[index] ?? "")) index += 1;
tokens.push(token("number", input.slice(start, index), start, index, depth));
continue;
}
if (WORD_START.test(ch)) {
index += ch.length;
while (index < input.length) {
const part = characterAt(input, index);
if (!WORD_PART.test(part) || (dialectId === "postgres" && part === "#")) break;
index += part.length;
}
tokens.push(token("word", input.slice(start, index), start, index, depth));
continue;
}
// Fullwidth () (U+FF08/U+FF09) are common IME typos in Chinese locales; treat them like
// ASCII () for nesting depth so commas inside to_date(…, …) stay nested and do not split
// INSERT ... SELECT projections (insert value column hints).
if ("(),.;*".includes(ch) || ch === "\uFF08" || ch === "\uFF09" || (options?.squareBracketsAsDelimiters && (ch === "[" || ch === "]"))) {
if (ch === ")" || ch === "\uFF09" || ch === "]") depth = Math.max(0, depth - 1);
tokens.push(token("punctuation", ch, start, start + ch.length, depth));
if (ch === "(" || ch === "\uFF08" || ch === "[") depth += 1;
index += ch.length;
continue;
}
index += ch.length;
tokens.push(token("operator", ch, start, index, depth));
}
return tokens;
}
export function tokenContainsPosition(tokenValue: SqlSemanticToken, position: number): boolean {
return tokenValue.span.start <= position && position <= tokenValue.span.end;
}
export function isSuppressedSqlSemanticContext(tokens: readonly SqlSemanticToken[], cursor: number): boolean {
return tokens.some((item) => (item.kind === "comment" || item.kind === "string") && item.span.start < cursor && cursor <= item.span.end);
}
export function findActiveSqlStatementSpan(sql: string, tokens: readonly SqlSemanticToken[], cursor: number): SqlSemanticSpan {
let start = 0;
let end = sql.length;
for (const item of tokens) {
if (item.kind !== "punctuation" || item.text !== ";" || item.depth !== 0) continue;
if (item.span.end <= cursor) start = item.span.end;
if (item.span.start >= cursor) {
end = item.span.start;
break;
}
}
while (start < end && /\s/.test(sql[start] ?? "")) start += 1;
while (end > start && /\s/.test(sql[end - 1] ?? "")) end -= 1;
return { start, end };
}
export function unquoteSqlSemanticIdentifier(tokenValue: SqlSemanticToken): string {
if (tokenValue.kind !== "quoted_identifier") return tokenValue.text;
if (tokenValue.quote === "[") return tokenValue.text.slice(1, -1).replaceAll("]]", "]");
const quote = tokenValue.quote ?? tokenValue.text[0] ?? "";
return tokenValue.text.slice(1, -1).replaceAll(quote + quote, quote);
}
export function tokenIsIdentifier(tokenValue: SqlSemanticToken | undefined): tokenValue is SqlSemanticToken {
return !!tokenValue && (tokenValue.kind === "word" || tokenValue.kind === "quoted_identifier");
}