:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(sql): add a format command to the SQL editor (#117279)
jake sciottod4da87c31e3b
diff --git a/frontend/package.json b/frontend/package.jsonindex 24c2967f72f6..487b7f11e9c4 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -304,6 +304,7 @@"rrule": "^2.8.1","simple-statistics": "^7.8.8","snappy-wasm": "^0.3.0",+ "sql-formatter": "^15.9.0","tailwind-merge": "^2.2.2","torph": "^0.0.9","ts-pattern": "catalog:",diff --git a/frontend/src/lib/monaco/formatHogQL.test.ts b/frontend/src/lib/monaco/formatHogQL.test.tsnew file mode 100644index 000000000000..1076e062d0c1--- /dev/null+++ b/frontend/src/lib/monaco/formatHogQL.test.ts@@ -0,0 +1,57 @@+import { MAX_NESTING_DEPTH, formatHogQL } from './formatHogQL'++describe('formatHogQL', () => {+ it.each([+ [+ 'HogQL-only syntax and the case of every token',+ "select properties.$browser, toStartOfDay(timestamp) as day, f'hi {event}' as greeting, a ?? b, c =~ 'x', any(d) from events where {filters} and {variables.n} = 1 // note",+ [+ 'select',+ ' properties.$browser,',+ ' toStartOfDay(timestamp) as day,',+ " f'hi {event}' as greeting,",+ ' a ?? b,',+ " c =~ 'x',",+ ' any(d)',+ 'from',+ ' events',+ 'where',+ ' {filters}',+ ' and {variables.n} = 1 // note',+ ],+ ],+ [+ 'every statement and a query commented out after the last one',+ 'select 1; select event from events;\n-- select 2;',+ ['select', ' 1;', '', 'select', ' event', 'from', ' events;', '', '-- select 2;'],+ ],+ ])('keeps %s', async (_name, query, lines) => {+ const { formatted } = await formatHogQL(query)++ expect(formatted).toEqual(lines.join('\n'))+ expect((await formatHogQL(formatted ?? '')).formatted).toEqual(formatted)+ })++ it('follows the editor indentation settings', async () => {+ expect((await formatHogQL('SELECT 1', { tabWidth: 2 })).formatted).toEqual('SELECT\n 1')+ expect((await formatHogQL('SELECT 1', { useTabs: true })).formatted).toEqual('SELECT\n\t1')+ })++ it.each([+ ['a query that does not parse', "SELECT 'unterminated FROM events", 'unparsable'],+ [+ 'a query nested too deeply to format quickly',+ `SELECT ${'('.repeat(MAX_NESTING_DEPTH + 1)}1${')'.repeat(MAX_NESTING_DEPTH + 1)}`,+ 'too_large',+ ],+ ['a string prefix the formatter would split from its quote', "SELECT E'\\n' FROM events", 'unsupported'],+ [+ 'a literal in a template string expression that the formatter would change',+ "SELECT f'{concat(' a b ')}'",+ 'unsupported',+ ],+ ['a HogQLX tag that the formatter would split', 'select <Sparkline data={[1,2,3]} />', 'unsupported'],+ ])('refuses %s', async (_name, query, failure) => {+ expect(await formatHogQL(query)).toEqual({ formatted: null, failure })+ })+})diff --git a/frontend/src/lib/monaco/formatHogQL.ts b/frontend/src/lib/monaco/formatHogQL.tsnew file mode 100644index 000000000000..86ceae491bfd--- /dev/null+++ b/frontend/src/lib/monaco/formatHogQL.ts@@ -0,0 +1,185 @@+import { type DialectOptions, clickhouse, expandPhrases, formatDialect } from 'sql-formatter'++import { retryImport } from 'lib/utils/retryImport'+import type { SelectComparison } from 'scenes/data-warehouse/editor/hogqlAst'++import { splitQueries } from './multiQueryUtils'++export interface FormatHogQLOptions {+ tabWidth?: number+ useTabs?: boolean+}++export type FormatHogQLFailure = 'too_large' | 'unparsable' | 'unsupported' | 'unverified'++export type FormatHogQLResult = { formatted: string; failure: null } | { formatted: null; failure: FormatHogQLFailure }++export const MAX_QUERY_LENGTH = 20_000+export const MAX_NESTING_DEPTH = 200++type Token = Parameters<NonNullable<DialectOptions['tokenizerOptions']['postProcess']>>[0][number]++const KEYWORD_TOKEN_TYPES = new Set<string>(['RESERVED_KEYWORD', 'RESERVED_DATA_TYPE'])++const KEYWORDS_FOLLOWED_BY_PARENS = new Set([+ 'AND',+ 'AS',+ 'BETWEEN',+ 'CASE',+ 'COHORT',+ 'DISTINCT',+ 'ELSE',+ 'END',+ 'EXISTS',+ 'ILIKE',+ 'IN',+ 'INTERVAL',+ 'IS',+ 'LIKE',+ 'NOT',+ 'ON',+ 'OR',+ 'OVER',+ 'THEN',+ 'USING',+ 'WHEN',+])++function keywordCallsAsFunctions(tokens: Token[]): Token[] {+ return tokens.map((token, index) => {+ const next = tokens[index + 1]+ const isCall = KEYWORD_TOKEN_TYPES.has(token.type) && next?.text === '(' && String(next.type) === 'OPEN_PAREN'+ return isCall && !KEYWORDS_FOLLOWED_BY_PARENS.has(token.text.toUpperCase())+ ? { ...token, type: 'RESERVED_FUNCTION_NAME' as Token['type'] }+ : token+ })+}++const hogQLDialect: DialectOptions = {+ name: 'hogql',+ tokenizerOptions: {+ ...clickhouse.tokenizerOptions,+ reservedClauses: [+ 'WITH',+ 'FROM',+ 'PREWHERE',+ 'WHERE',+ 'GROUP BY',+ 'HAVING',+ 'PARTITION BY',+ 'ORDER BY',+ 'LIMIT',+ 'SETTINGS',+ ],+ reservedSetOperations: expandPhrases(['{UNION | INTERSECT | EXCEPT} [ALL | DISTINCT] [BY NAME]']),+ reservedJoins: expandPhrases([+ '[GLOBAL] [INNER | LEFT | RIGHT | FULL | CROSS] [OUTER] [SEMI | ANTI | ANY | ALL | ASOF] JOIN',+ '[GLOBAL] [SEMI | ANTI | ANY | ALL | ASOF] [INNER | LEFT | RIGHT | FULL] [OUTER] JOIN',+ '[LEFT] ARRAY JOIN',+ ]),+ reservedKeywordPhrases: expandPhrases(['WITH {TOTALS | TIES | CUBE | ROLLUP | FILL}']),+ identChars: { first: '$', rest: '$' },+ stringTypes: [{ quote: "''-qq-bs", prefixes: ['f', 'F'] }],+ lineCommentTypes: ['--', '//', '#'],+ paramTypes: { custom: [{ regex: String.raw`\{[^{}]*\}` }] },+ operators: [...(clickhouse.tokenizerOptions.operators ?? []), '??', '~', '=~', '!~', '~*', '=~*', '!~*'],+ postProcess: keywordCallsAsFunctions,+ },+ formatOptions: { onelineClauses: [], tabularOnelineClauses: [] },+}++function nestingDepth(query: string): number {+ let depth = 0+ let deepest = 0+ for (const char of query) {+ if (char === '(') {+ depth += 1+ deepest = Math.max(deepest, depth)+ } else if (char === ')') {+ depth = Math.max(0, depth - 1)+ }+ }+ return deepest+}++const TOKEN_CHARACTER = /[\w$'"`]/++function significantText(query: string): string {+ let text = ''+ let pendingSpace = false+ for (const char of query) {+ if (/\s/.test(char)) {+ pendingSpace = true+ continue+ }+ const previous = text[text.length - 1]+ if (pendingSpace && previous !== undefined && TOKEN_CHARACTER.test(previous) && TOKEN_CHARACTER.test(char)) {+ text += ' '+ }+ pendingSpace = false+ text += char+ }+ return text+}++const COMMENT = /(?:--|\/\/)[^\n]*|#(?![0-9])[^\n]*|\/\*[\s\S]*?\*\//g++function statementsOf(query: string): string[] {+ return splitQueries(query, { hogql: true })+ .map(({ query: statement }) => statement)+ .filter((statement) => statement.replace(COMMENT, '').trim() !== '')+}++async function verifyStatements(query: string, formatted: string): Promise<FormatHogQLFailure | null> {+ const statements = statementsOf(query)+ const formattedStatements = statementsOf(formatted)+ if (statements.length !== formattedStatements.length) {+ return 'unsupported'+ }+ let comparisons: SelectComparison[]+ try {+ const { compareSelectQueries } = await retryImport(+ () => import('scenes/data-warehouse/editor/hogqlParserWorkerManager')+ )+ comparisons = await Promise.all(+ statements.map((statement, index) => compareSelectQueries(statement, formattedStatements[index]))+ )+ } catch {+ return 'unverified'+ }+ if (comparisons.includes('unparsable')) {+ return 'unparsable'+ }+ return comparisons.every((comparison) => comparison === 'same') ? null : 'unsupported'+}++export async function formatHogQL(+ query: string,+ { tabWidth = 4, useTabs = false }: FormatHogQLOptions = {}+): Promise<FormatHogQLResult> {+ if (query.length > MAX_QUERY_LENGTH || nestingDepth(query) > MAX_NESTING_DEPTH) {+ return { formatted: null, failure: 'too_large' }+ }+ let formatted: string+ try {+ formatted = formatDialect(query, {+ dialect: hogQLDialect,+ tabWidth,+ useTabs,+ keywordCase: 'preserve',+ dataTypeCase: 'preserve',+ functionCase: 'preserve',+ identifierCase: 'preserve',+ })+ } catch {+ return { formatted: null, failure: 'unparsable' }+ }+ if (significantText(formatted) !== significantText(query)) {+ return { formatted: null, failure: 'unsupported' }+ }+ if (formatted === query) {+ return { formatted, failure: null }+ }+ const failure = await verifyStatements(query, formatted)+ return failure === null ? { formatted, failure: null } : { formatted: null, failure }+}diff --git a/frontend/src/lib/monaco/hogQLAutocompleteProvider.ts b/frontend/src/lib/monaco/hogQLAutocompleteProvider.tsindex d7efd54993c2..2ff20b352260 100644--- a/frontend/src/lib/monaco/hogQLAutocompleteProvider.ts+++ b/frontend/src/lib/monaco/hogQLAutocompleteProvider.ts@@ -120,8 +120,11 @@ export const hogQLAutocompleteProvider = (type: HogLanguage): languages.Completi// `;`-separated queries fails to parse and comes back with no suggestions at all.// Send just the statement under the cursor, with the offsets rebased onto it.// Single-statement documents are still sent whole so a trailing `;` behaves as before.- const statements = type === HogLanguage.hogQL ? splitQueries(queryText) : []- const statement = statements.length > 1 ? findStatementAtOffset(queryText, model.getOffsetAt(position)) : null+ const statements = type === HogLanguage.hogQL ? splitQueries(queryText, { hogql: true }) : []+ const statement =+ statements.length > 1+ ? findStatementAtOffset(queryText, model.getOffsetAt(position), { hogql: true })+ : nullconst statementOffset = statement?.start ?? 0const query: HogQLAutocomplete = setLatestVersionsOnQuery(diff --git a/frontend/src/lib/monaco/hogQLFormattingProvider.test.ts b/frontend/src/lib/monaco/hogQLFormattingProvider.test.tsnew file mode 100644index 000000000000..3cc7bd6d4839--- /dev/null+++ b/frontend/src/lib/monaco/hogQLFormattingProvider.test.ts@@ -0,0 +1,32 @@+import { editor, languages } from 'monaco-editor'+import posthog from 'posthog-js'++import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'++import { hogQLFormattingProvider } from './hogQLFormattingProvider'++jest.mock('monaco-editor', () => ({ editor: { EndOfLinePreference: { LF: 1 } } }))+jest.mock('posthog-js')+jest.mock('lib/lemon-ui/LemonToast/LemonToast', () => ({+ lemonToast: { error: jest.fn() },+}))+jest.mock('./formatHogQL', () => {+ throw new Error('Failed to load the formatter')+})++describe('hogQLFormattingProvider', () => {+ it('keeps the query and shows a toast when the formatter does not load', async () => {+ const model = { getValue: () => 'select 1' } as unknown as editor.ITextModel+ const options = { tabSize: 4, insertSpaces: true } as languages.FormattingOptions++ const edits = await hogQLFormattingProvider().provideDocumentFormattingEdits(model, options, {} as any)++ expect(edits).toEqual([])+ expect(lemonToast.error).toHaveBeenCalledWith("Couldn't format this query. Try again in a moment.")+ expect(posthog.captureException).toHaveBeenCalled()+ expect(posthog.capture).toHaveBeenCalledWith(+ 'sql-editor-query-formatted',+ expect.objectContaining({ succeeded: false, failure: 'unverified' })+ )+ })+})diff --git a/frontend/src/lib/monaco/hogQLFormattingProvider.ts b/frontend/src/lib/monaco/hogQLFormattingProvider.tsnew file mode 100644index 000000000000..2890b185f8e9--- /dev/null+++ b/frontend/src/lib/monaco/hogQLFormattingProvider.ts@@ -0,0 +1,68 @@+import type { Monaco } from '@monaco-editor/react'+import type { BuiltLogic } from 'kea'+import { editor, languages } from 'monaco-editor'+import posthog from 'posthog-js'++import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'+import { retryImport } from 'lib/utils/retryImport'++import { HogLanguage } from '~/queries/schema/schema-general'++import type { codeEditorLogicType } from './codeEditorLogic'+import type { FormatHogQLFailure, FormatHogQLResult } from './formatHogQL'++const FAILURE_MESSAGES: Record<FormatHogQLFailure | 'raw_sql', string> = {+ raw_sql: 'Formatting is only available for HogQL queries. Turn off "Send raw query" to format this one.',+ too_large: 'This query is too long or too deeply nested to format. Split it into smaller queries.',+ unparsable: "Couldn't format this query. Check it for syntax errors and try again.",+ unsupported: "Couldn't format this query. It uses syntax the formatter does not support yet.",+ unverified: "Couldn't format this query. Try again in a moment.",+}++function isRawSqlModel(model: editor.ITextModel): boolean {+ const logic: BuiltLogic<codeEditorLogicType> | undefined = (model as any).codeEditorLogic+ const sourceQuery = logic?.props.sourceQuery+ return !!sourceQuery && 'sendRawQuery' in sourceQuery && !!sourceQuery.sendRawQuery+}++async function formatQuery(query: string, options: languages.FormattingOptions): Promise<FormatHogQLResult> {+ try {+ const { formatHogQL } = await retryImport(() => import('./formatHogQL'))+ return await formatHogQL(query, { tabWidth: options.tabSize, useTabs: !options.insertSpaces })+ } catch (error) {+ posthog.captureException(error)+ return { formatted: null, failure: 'unverified' }+ }+}++export const hogQLFormattingProvider: () => languages.DocumentFormattingEditProvider = () => ({+ provideDocumentFormattingEdits: async (model, options) => {+ const query = model.getValue(editor.EndOfLinePreference.LF)+ const result = isRawSqlModel(model)+ ? ({ formatted: null, failure: 'raw_sql' } as const)+ : await formatQuery(query, options)+ posthog.capture('sql-editor-query-formatted', {+ succeeded: result.failure === null,+ changed: result.formatted !== null && result.formatted !== query,+ failure: result.failure,+ })+ if (result.failure !== null) {+ lemonToast.error(FAILURE_MESSAGES[result.failure])+ return []+ }+ if (result.formatted === query) {+ return []+ }+ return [{ range: model.getFullModelRange(), text: result.formatted }]+ },+})++const monacosWithFormatting = new WeakSet<Monaco>()++export function registerHogQLFormattingProvider(monaco: Monaco): void {+ if (monacosWithFormatting.has(monaco)) {+ return+ }+ monacosWithFormatting.add(monaco)+ monaco.languages.registerDocumentFormattingEditProvider(HogLanguage.hogQL, hogQLFormattingProvider())+}diff --git a/frontend/src/lib/monaco/multiQueryUtils.test.ts b/frontend/src/lib/monaco/multiQueryUtils.test.tsindex 9c7396459aa4..2b39db17a7e5 100644--- a/frontend/src/lib/monaco/multiQueryUtils.test.ts+++ b/frontend/src/lib/monaco/multiQueryUtils.test.ts@@ -80,15 +80,25 @@ FROM persons`expect(result[0].query).toContain("it\\'s;here")})- it('ignores semicolons inside single-line comments', () => {- const input = `SELECT 1 -- this; is a comment+ it.each(['//', '#'])('ignores semicolons inside HogQL %s comments', (comment) => {+ const input = `SELECT 1 ${comment} this; is a comment; SELECT 2`- const result = splitQueries(input)+ const result = splitQueries(input, { hogql: true })expect(result).toHaveLength(2)expect(result[0].query).toContain('SELECT 1')expect(result[1].query).toBe('SELECT 2')})+ it('does not treat positional references as HogQL hash comments', () => {+ const result = splitQueries('SELECT 1 ORDER BY #1; SELECT 2', { hogql: true })+ expect(result).toHaveLength(2)+ })++ it('does not treat a hash as a raw SQL comment', () => {+ const result = splitQueries('SELECT 5 # 3; SELECT 2')+ expect(result).toHaveLength(2)+ })+it('ignores semicolons inside block comments', () => {const result = splitQueries('SELECT 1 /* ; not a split */ ; SELECT 2')expect(result).toHaveLength(2)diff --git a/frontend/src/lib/monaco/multiQueryUtils.ts b/frontend/src/lib/monaco/multiQueryUtils.tsindex 4b580634a2e4..ff8665ae4c22 100644--- a/frontend/src/lib/monaco/multiQueryUtils.ts+++ b/frontend/src/lib/monaco/multiQueryUtils.ts@@ -7,11 +7,15 @@ export interface QueryRange {end: number}+interface SplitQueryOptions {+ hogql?: boolean+}+/*** Offsets of the semicolons that separate top-level statements, skipping those inside string* literals, comments, and parenthesized blocks (subqueries, CTEs, function calls).*/-function findSeparatorOffsets(input: string): number[] {+function findSeparatorOffsets(input: string, hogql: boolean): number[] {const separators: number[] = []let i = 0let parenDepth = 0@@ -50,9 +54,10 @@ function findSeparatorOffsets(input: string): number[] {continue}- // Skip single-line comments- if (ch === '-' && input[i + 1] === '-') {- i += 2+ const isLineComment =+ (ch === '-' && input[i + 1] === '-') ||+ (hogql && ((ch === '/' && input[i + 1] === '/') || (ch === '#' && !/[0-9]/.test(input[i + 1] ?? ''))))+ if (isLineComment) {while (i < input.length && input[i] !== '\n') {i++}@@ -105,13 +110,13 @@ function findSeparatorOffsets(input: string): number[] {* Each range covers the trimmed statement text, so the whitespace around a statement sits* outside it. Segments holding no statement text are dropped.*/-export function splitQueries(input: string): QueryRange[] {+export function splitQueries(input: string, { hogql = false }: SplitQueryOptions = {}): QueryRange[] {if (!input.trim()) {return []}const ranges: QueryRange[] = []- for (const { start, end } of statementSegments(input)) {+ for (const { start, end } of statementSegments(input, hogql)) {const segment = input.slice(start, end)const trimmed = segment.trim()if (trimmed) {@@ -128,10 +133,10 @@ export function splitQueries(input: string): QueryRange[] {* each statement. A cursor at `end` sits on the separator itself, which still belongs to the* statement before it.*/-function statementSegments(input: string): { start: number; end: number }[] {+function statementSegments(input: string, hogql = false): { start: number; end: number }[] {const segments: { start: number; end: number }[] = []let start = 0- for (const separator of findSeparatorOffsets(input)) {+ for (const separator of findSeparatorOffsets(input, hogql)) {segments.push({ start, end: separator })start = separator + 1}@@ -149,8 +154,14 @@ function statementSegments(input: string): { start: number; end: number }[] {* Returns null when the cursor sits in a segment that holds no statement text yet, such as directly* after a semicolon.*/-export function findStatementAtOffset(input: string, cursorOffset: number): QueryRange | null {- const segment = statementSegments(input).find(({ start, end }) => cursorOffset >= start && cursorOffset <= end)+export function findStatementAtOffset(+ input: string,+ cursorOffset: number,+ { hogql = false }: SplitQueryOptions = {}+): QueryRange | null {+ const segment = statementSegments(input, hogql).find(+ ({ start, end }) => cursorOffset >= start && cursorOffset <= end+ )if (!segment || !input.slice(segment.start, segment.end).trim()) {return null}diff --git a/frontend/src/scenes/data-warehouse/editor/QueryWindow.tsx b/frontend/src/scenes/data-warehouse/editor/QueryWindow.tsxindex ed7e0984c1a7..25df16963365 100644--- a/frontend/src/scenes/data-warehouse/editor/QueryWindow.tsx+++ b/frontend/src/scenes/data-warehouse/editor/QueryWindow.tsx@@ -14,6 +14,7 @@ import { LemonButton } from 'lib/lemon-ui/LemonButton'import { LemonMenu } from 'lib/lemon-ui/LemonMenu/LemonMenu'import { LemonSwitch } from 'lib/lemon-ui/LemonSwitch'import { Tooltip } from 'lib/lemon-ui/Tooltip'+import { registerHogQLFormattingProvider } from 'lib/monaco/hogQLFormattingProvider'import { cn } from 'lib/utils/css-classes'import { SQLEditorMode } from 'scenes/data-warehouse/editor/sqlEditorModes'import { urls } from 'scenes/urls'@@ -401,6 +402,7 @@ export function QueryWindow({setQueryInput(v ?? '')},onMount: (editor, monaco) => {+ registerHogQLFormattingProvider(monaco)onSetMonacoAndEditor(monaco, editor)},onPressCmdEnter: (value, selectionType) => {diff --git a/frontend/src/scenes/data-warehouse/editor/hogqlAst.ts b/frontend/src/scenes/data-warehouse/editor/hogqlAst.tsindex 53a17fea429d..d79196f1ba8c 100644--- a/frontend/src/scenes/data-warehouse/editor/hogqlAst.ts+++ b/frontend/src/scenes/data-warehouse/editor/hogqlAst.ts@@ -147,3 +147,12 @@ export function tablesAndColumnsFromAst(ast: ASTNode): Record<string, Record<strreturn selectedColumnsByTable}++export type SelectComparison = 'same' | 'different' | 'unparsable'++export function compareParsedSelects(parsed: string, otherParsed: string): SelectComparison {+ if ((JSON.parse(parsed) as { error?: boolean }).error) {+ return 'unparsable'+ }+ return parsed === otherParsed ? 'same' : 'different'+}diff --git a/frontend/src/scenes/data-warehouse/editor/hogqlParserWorker.ts b/frontend/src/scenes/data-warehouse/editor/hogqlParserWorker.tsindex dc918d8363cb..f10b14b694ec 100644--- a/frontend/src/scenes/data-warehouse/editor/hogqlParserWorker.ts+++ b/frontend/src/scenes/data-warehouse/editor/hogqlParserWorker.ts@@ -1,6 +1,12 @@import type { ASTNode } from '@posthog/hogql-parser'-import { type AstRange, innermostSelectRangeFromAst, tablesAndColumnsFromAst } from './hogqlAst'+import {+ type AstRange,+ type SelectComparison,+ compareParsedSelects,+ innermostSelectRangeFromAst,+ tablesAndColumnsFromAst,+} from './hogqlAst'// Parsing HogQL is a synchronous WASM call over the whole query — roughly 36ms per 1000// characters, so seconds on a long one. On the main thread that freezes the editor after every@@ -9,12 +15,13 @@ import { type AstRange, innermostSelectRangeFromAst, tablesAndColumnsFromAst } fexport type HogqlParserQuestion =| { op: 'innermostSelect'; query: string; localOffset: number }| { op: 'tablesAndColumns'; query: string }+ | { op: 'compareSelects'; query: string; other: string }export type HogqlParserRequest = HogqlParserQuestion & { id: number }export type HogqlParserResponse =| { type: 'ready' }- | { id: number; result: AstRange | null | Record<string, Record<string, boolean>> }+ | { id: number; result: AstRange | null | Record<string, Record<string, boolean>> | SelectComparison }| { id: number; error: string }let parserPromise: Promise<{ parseSelect: (input: string, isInternal?: boolean) => string }> | null = null@@ -57,6 +64,15 @@ async function astFor(query: string): Promise<ASTNode | null> {self.onmessage = async (event: MessageEvent<HogqlParserRequest>): Promise<void> => {const request = event.datatry {+ if (request.op === 'compareSelects') {+ const parser = await getParser()+ const result = compareParsedSelects(+ parser.parseSelect(request.query, true),+ parser.parseSelect(request.other, true)+ )+ self.postMessage({ id: request.id, result } satisfies HogqlParserResponse)+ return+ }const ast = await astFor(request.query)if (!ast) {self.postMessage({diff --git a/frontend/src/scenes/data-warehouse/editor/hogqlParserWorkerManager.ts b/frontend/src/scenes/data-warehouse/editor/hogqlParserWorkerManager.tsindex 489884def8fb..85654fbe17cf 100644--- a/frontend/src/scenes/data-warehouse/editor/hogqlParserWorkerManager.ts+++ b/frontend/src/scenes/data-warehouse/editor/hogqlParserWorkerManager.ts@@ -1,4 +1,10 @@-import { type AstRange, innermostSelectRangeFromAst, tablesAndColumnsFromAst } from './hogqlAst'+import {+ type AstRange,+ type SelectComparison,+ compareParsedSelects,+ innermostSelectRangeFromAst,+ tablesAndColumnsFromAst,+} from './hogqlAst'import { parseSelect } from './hogqlParserSingleton'import type { HogqlParserQuestion, HogqlParserRequest, HogqlParserResponse } from './hogqlParserWorker'@@ -136,3 +142,17 @@ export async function analyzeTablesAndColumns(query: string): Promise<Record<strconst ast = await parseOnMainThread(query)return ast ? tablesAndColumnsFromAst(ast) : {}}++const SELECT_COMPARISONS: ReadonlySet<unknown> = new Set<SelectComparison>(['same', 'different', 'unparsable'])++export async function compareSelectQueries(query: string, other: string): Promise<SelectComparison> {+ if (!workerUnavailable) {+ try {+ const result = await ask({ op: 'compareSelects', query, other })+ if (SELECT_COMPARISONS.has(result)) {+ return result+ }+ } catch {}+ }+ return compareParsedSelects(await parseSelect(query, true), await parseSelect(other, true))+}diff --git a/playwright/e2e/sql-editor.spec.ts b/playwright/e2e/sql-editor.spec.tsindex abcfa93ea01c..eb631afcae84 100644--- a/playwright/e2e/sql-editor.spec.ts+++ b/playwright/e2e/sql-editor.spec.ts@@ -125,6 +125,58 @@ test.describe('SQL Editor', () => {await expect(page.getByTestId('sql-editor-sidebar-query-info-pane')).toBeVisible()})+ test('Format query', async ({ page }) => {+ const sqlInsight = new SqlInsight(page)+ const oneLineQuery =+ "select event, count() as total from events where event = '$pageview' group by event order by total desc limit 10"+ const formattedQuery = [+ 'select',+ ' event,',+ ' count() as total',+ 'from',+ ' events',+ 'where',+ " event = '$pageview'",+ 'group by',+ ' event',+ 'order by',+ ' total desc',+ 'limit',+ ' 10',+ ].join('\n')+ const readEditorText = (): Promise<string> =>+ page.evaluate(+ () => (window as any).__monacoEditors?.at(-1)?.getModel()?.getValue().replace(/\r\n/g, '\n') ?? ''+ )++ const editorArea = page.getByTestId('hogql-query-editor')++ await test.step('the keyboard shortcut reflows the query', async () => {+ await sqlInsight.writeQuery(oneLineQuery)+ await editorArea.click()+ await page.keyboard.press('Shift+Alt+F')+ await expect.poll(readEditorText).toEqual(formattedQuery)+ })++ await test.step('the context menu command reflows the query', async () => {+ await sqlInsight.writeQuery(oneLineQuery)+ await editorArea.locator('.view-lines').click({ button: 'right' })+ const menuItem = page.getByRole('menuitem', { name: 'Format Document' })+ await expect(menuItem).toBeVisible()+ await page.waitForTimeout(250)+ await menuItem.click()+ await expect.poll(readEditorText).toEqual(formattedQuery)+ })++ await test.step('a query that does not parse keeps its text', async () => {+ await sqlInsight.writeQuery('select ) from events')+ await editorArea.click()+ await page.keyboard.press('Shift+Alt+F')+ await expect(page.getByText("Couldn't format this query.", { exact: false })).toBeVisible()+ await expect.poll(readEditorText).toEqual('select ) from events')+ })+ })+test('Query variables pane', async ({ page }) => {await page.getByTestId('sql-editor-variables-button').click()await expect(page.getByPlaceholder('Search variables')).toBeVisible({ timeout: 60000 })diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 8a05c0c18748..37bc3d543504 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1288,6 +1288,9 @@ importers:snappy-wasm:specifier: ^0.3.0version: 0.3.0+ sql-formatter:+ specifier: ^15.9.0+ version: 15.9.0tailwind-merge:specifier: ^2.2.2version: 2.2.2@@ -16369,6 +16372,9 @@ packages:resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}engines: {node: '>=8'}+ resolution: {integrity: sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==}+resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}engines: {node: '>=0.10.0'}@@ -19778,6 +19784,9 @@ packages:resolution: {integrity: sha512-i/+ZKXMDf6aqYtBhuOcej71YSlbjT3wCO/4H1j8rPvxDJEifdwgg5MaFyu6iYAT8GBZJg2z0dkgK4YMzvURALQ==}+ resolution: {integrity: sha512-m2fmM2dDm7GZQsY7KK2cme8agi+AAljILjQnof7p1ZMDe6dQ4bdnSMx0cPppudoeNv5hEFQirN6u+O4fDE0IWA==}+resolution: {integrity: sha512-DlWOmsXMJrV8lzZyd+LKjG2CXULUs++bkq8GZ2Sr0R0RRhs30K2wtY+LKiTjhmJU3W61HK+rB0GLz6XmPvTA1A==}@@ -19880,6 +19889,10 @@ packages:resolution: {integrity: sha512-x7ZdOwBxZCEm9MM7+eQCjkrNLrW3rkBKNHVr78zbtqnMGVNlnDi6C/eUEYgxHNrcbu0ymvjzcwIL/6H1iHri9g==}engines: {node: '>=18'}+ resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==}+ hasBin: true+resolution: {integrity: sha512-6k0YULvhpw+RoLNiQCRKOl09Rv1dPLr8hHnVjHqdolKwDrdNyk+Hmrthi4lIGPPz3r39dLx0hsF5s40sZ3Us4Q==}engines: {node: '>= 4.4.x'}@@ -21268,6 +21281,13 @@ packages:resolution: {integrity: sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==}+ resolution: {integrity: sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==}++ resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==}+ engines: {node: '>=0.12'}+resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}@@ -21907,6 +21927,10 @@ packages:resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==}engines: {node: '>=18'}+ resolution: {integrity: sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==}+ engines: {node: '>=0.12'}+resolution: {integrity: sha512-xA93uxUD/rogV7BV59agW/JHPGXeREMWiZc9jhcwY4YdZ7QOtC7qbomYg0n4wyk2lJhggjvKvhNX8wln/Aldhg==}engines: {node: '>=8.10.0'}@@ -22440,6 +22464,10 @@ packages:resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==}+ resolution: {integrity: sha512-3A/N+v+mA5/PDU+FlO4uTFE0L/Mmg63c/Ny161Qv2b1Mxp9IX8War+LLmq17LA/Kgu4KXAR7iutK2JaHWJMMtw==}+ hasBin: true+resolution: {integrity: sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==}engines: {node: '>=16.20.2'}@@ -36755,6 +36783,8 @@ snapshots:dependencies:path-type: 4.0.0+ [email protected]: {}+dependencies:esutils: 2.0.3@@ -41876,6 +41906,8 @@ snapshots:dependencies:color-name: 1.1.4+ [email protected]: {}+dependencies:motion-utils: 12.24.10@@ -42028,6 +42060,13 @@ snapshots:+ dependencies:+ commander: 2.20.3+ moo: 0.5.3+ railroad-diagrams: 1.0.0+ randexp: 0.4.6+dependencies:iconv-lite: 0.6.3@@ -43888,6 +43927,13 @@ snapshots:+ [email protected]: {}++ dependencies:+ discontinuous-range: 1.0.0+ ret: 0.1.15+dependencies:safe-buffer: 5.2.1@@ -44724,6 +44770,8 @@ snapshots:onetime: 7.0.0signal-exit: 4.1.0+ [email protected]: {}+dependencies:debug: 4.4.3([email protected])@@ -45365,6 +45413,11 @@ snapshots:+ dependencies:+ argparse: 2.0.1+ nearley: 2.20.1+dependencies:concat-stream: 2.0.0