Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(studio): load ClickHouse query templates when OTEL logs are on (#47390)

Jordi Enricb9f76d95f796
diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts
index ec5fc44dbaf..ab6671faf79 100644
--- a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts
+++ b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts
@@ -286,6 +286,157 @@ limit 100;
},
]
+// ClickHouse rewrites of the custom BigQuery templates above, used when the OTEL
+// logs engine is on. Keyed by template label. Simple-mode templates are plain
+// event_message searches that work unchanged, so they have no entry here.
+const OTEL_TEMPLATE_SEARCH_STRINGS: Record<string, string> = {
+ 'Commits By User': `select
+ log_attributes['parsed.user_name'] as user_name,
+ count() as count
+from logs
+where source = 'postgres_logs'
+ and event_message like '%COMMIT%'
+group by user_name
+order by count desc
+limit 100`,
+ 'Metadata IP': `select
+ timestamp,
+ log_attributes['request.headers.x_real_ip'] as x_real_ip
+from logs
+where source = 'edge_logs'
+ and log_attributes['request.headers.x_real_ip'] != ''
+order by timestamp desc
+limit 100`,
+ 'Requests by Geography': `select
+ log_attributes['request.cf.country'] as country,
+ count() as count
+from logs
+where source = 'edge_logs'
+group by country
+order by count desc
+limit 100`,
+ 'Slow Response Time': `select
+ timestamp,
+ event_message,
+ toInt32OrZero(log_attributes['response.origin_time']) as origin_time
+from logs
+where source = 'edge_logs'
+ and toInt32OrZero(log_attributes['response.origin_time']) > 1000
+order by timestamp desc
+limit 100`,
+ '500 Request Codes': `select
+ timestamp,
+ event_message,
+ toInt32OrZero(log_attributes['response.status_code']) as status_code
+from logs
+where source = 'edge_logs'
+ and toInt32OrZero(log_attributes['response.status_code']) >= 500
+order by timestamp desc
+limit 100`,
+ 'Top Paths': `select
+ log_attributes['request.path'] as path,
+ log_attributes['request.search'] as params,
+ count() as c
+from logs
+where source = 'edge_logs'
+group by path, params
+order by c desc
+limit 100`,
+ 'REST Requests': `select
+ timestamp,
+ event_message
+from logs
+where source = 'edge_logs'
+ and log_attributes['request.path'] like '%rest/v1%'
+order by timestamp desc
+limit 100`,
+ Errors: `select
+ timestamp,
+ log_attributes['parsed.error_severity'] as error_severity,
+ event_message
+from logs
+where source = 'postgres_logs'
+ and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')
+order by timestamp desc
+limit 100`,
+ 'Error Count by User': `select
+ count() as count,
+ log_attributes['parsed.user_name'] as user_name,
+ log_attributes['parsed.error_severity'] as error_severity
+from logs
+where source = 'postgres_logs'
+ and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')
+group by user_name, error_severity
+order by count desc
+limit 100`,
+ 'Auth Endpoint Events': `select
+ timestamp,
+ event_message
+from logs
+where source = 'auth_logs'
+ and match(event_message, 'level.{3}(info|warning|error|fatal)')
+order by timestamp desc
+limit 100`,
+ 'Auth Audit Logs': `select
+ timestamp,
+ event_message,
+ log_attributes
+from logs
+where source = 'auth_audit_logs'
+order by timestamp desc
+limit 10`,
+ 'Storage Object Requests': `select
+ log_attributes['request.method'] as http_verb,
+ log_attributes['request.path'] as filepath,
+ count() as num_requests
+from logs
+where source = 'edge_logs'
+ and log_attributes['request.path'] like '%storage/v1/object/%'
+group by http_verb, filepath
+order by num_requests desc
+limit 100`,
+ 'Storage Egress Requests': `select
+ log_attributes['request.method'] as http_verb,
+ log_attributes['request.path'] as filepath,
+ (log_attributes['response.headers.cf_cache_status'] = 'HIT') as cached,
+ count() as num_requests
+from logs
+where source = 'edge_logs'
+ and (
+ log_attributes['request.path'] like '%storage/v1/object/%'
+ or log_attributes['request.path'] like '%storage/v1/render/%'
+ )
+ and log_attributes['request.method'] = 'GET'
+group by http_verb, filepath, cached
+order by num_requests desc
+limit 100`,
+ 'Storage Top Cache Misses': `select
+ log_attributes['request.path'] as path,
+ log_attributes['request.search'] as search,
+ count() as count
+from logs
+where source = 'edge_logs'
+ and startsWith(log_attributes['request.path'], '/storage/v1/object')
+ and log_attributes['request.method'] = 'GET'
+ and log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC')
+group by path, search
+order by count desc
+limit 100`,
+}
+
+/**
+ * Returns the log templates for the active engine. On the OTEL/ClickHouse engine
+ * the custom templates, written for BigQuery, are swapped for their ClickHouse
+ * rewrites; everything else is returned unchanged.
+ */
+export function getLogsTemplates(useOtel: boolean): LogTemplate[] {
+ if (!useOtel) return TEMPLATES
+ return TEMPLATES.map((template) => {
+ const otelSearchString = template.label && OTEL_TEMPLATE_SEARCH_STRINGS[template.label]
+ return otelSearchString ? { ...template, searchString: otelSearchString } : template
+ })
+}
+
type SqlFilterFn = (value: any) => SafeLogSqlFragment
export type SqlFilterEntry = SafeLogSqlFragment | SqlFilterFn
diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts
new file mode 100644
index 00000000000..f0ee03d2768
--- /dev/null
+++ b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest'
+
+import { getLogsTemplates, TEMPLATES } from './Logs.constants'
+
+const customTemplates = TEMPLATES.filter((t) => t.mode === 'custom')
+const bigQueryConstructs =
+ /unnest|cross join|left join|regexp_contains|cast\(\s*timestamp|count\(\*\)/i
+
+describe('getLogsTemplates', () => {
+ it('returns the original templates when the OTEL engine is off', () => {
+ expect(getLogsTemplates(false)).toBe(TEMPLATES)
+ })
+
+ it('gives every custom template a ClickHouse rewrite when OTEL is on', () => {
+ const otel = getLogsTemplates(true)
+ for (const template of customTemplates) {
+ const rewritten = otel.find((t) => t.label === template.label)
+ expect(rewritten, `missing OTEL rewrite for "${template.label}"`).toBeDefined()
+ expect(
+ rewritten!.searchString,
+ `"${template.label}" was not rewritten for ClickHouse`
+ ).not.toBe(template.searchString)
+ }
+ })
+
+ it('produces ClickHouse SQL with no BigQuery constructs', () => {
+ const otel = getLogsTemplates(true)
+ for (const template of otel.filter((t) => t.mode === 'custom')) {
+ expect(template.searchString, `"${template.label}" still targets logs table`).toMatch(
+ /from logs/i
+ )
+ expect(template.searchString, `"${template.label}" filters by source`).toMatch(/source\s*=/i)
+ expect(
+ template.searchString,
+ `"${template.label}" still contains a BigQuery construct`
+ ).not.toMatch(bigQueryConstructs)
+ }
+ })
+
+ it('leaves simple-mode templates unchanged on the OTEL engine', () => {
+ const otel = getLogsTemplates(true)
+ for (const template of TEMPLATES.filter((t) => t.mode === 'simple')) {
+ const same = otel.find((t) => t.label === template.label)
+ expect(same!.searchString).toBe(template.searchString)
+ }
+ })
+
+ it('maps nested BigQuery fields to the right log_attributes keys', () => {
+ const otel = getLogsTemplates(true)
+ const byLabel = (label: string) => otel.find((t) => t.label === label)!.searchString
+
+ expect(byLabel('Requests by Geography')).toContain("log_attributes['request.cf.country']")
+ expect(byLabel('Metadata IP')).toContain("log_attributes['request.headers.x_real_ip']")
+ expect(byLabel('Errors')).toContain("log_attributes['parsed.error_severity']")
+ expect(byLabel('Slow Response Time')).toContain(
+ "toInt32OrZero(log_attributes['response.origin_time'])"
+ )
+ })
+})
diff --git a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx
index 1fb54164151..c246e2da385 100644
--- a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx
+++ b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx
@@ -16,8 +16,8 @@ import {
import {
EXPLORER_DATEPICKER_HELPERS,
getDefaultHelper,
+ getLogsTemplates,
LOGS_LARGE_DATE_RANGE_DAYS_THRESHOLD,
- TEMPLATES,
} from '@/components/interfaces/Settings/Logs/Logs.constants'
import { DatePickerValue } from '@/components/interfaces/Settings/Logs/Logs.DatePickers'
import { LogData, LogsWarning, LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'
@@ -85,11 +85,13 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
const track = useTrack()
const projectRef = ref as string
const { logsShowMetadataIpTemplate } = useIsFeatureEnabled(['logs:show_metadata_ip_template'])
+ const useOtelEndpoint = useFlag('otelLegacyLogs')
const allTemplates = useMemo(() => {
- if (logsShowMetadataIpTemplate) return TEMPLATES
- else return TEMPLATES.filter((x) => x.label !== 'Metadata IP')
- }, [logsShowMetadataIpTemplate])
+ const templates = getLogsTemplates(useOtelEndpoint)
+ if (logsShowMetadataIpTemplate) return templates
+ else return templates.filter((x) => x.label !== 'Metadata IP')
+ }, [logsShowMetadataIpTemplate, useOtelEndpoint])
const editorRef = useRef<editor.IStandaloneCodeEditor>(null)
const [editorId] = useState<string>(uuidv4())
@@ -111,8 +113,6 @@ export const LogsExplorerPage: NextPageWithLayout = () => {
}, [timestampStart, timestampEnd, defaultHelper])
const [datePickerValue, setDatePickerValue] = useState<DatePickerValue>(initialDatePickerValue)
- const useOtelEndpoint = useFlag('otelLegacyLogs')
-
const { logsDefaultQuery } = useCustomContent(['logs:default_query'])
const PLACEHOLDER_QUERY = useOtelEndpoint
? OTEL_PLACEHOLDER_QUERY
diff --git a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx
index d0f507ed041..d5af19ee7cd 100644
--- a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx
+++ b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx
@@ -1,9 +1,9 @@
-import { useParams } from 'common'
+import { useFlag, useParams } from 'common'
import { CodeIcon } from 'lucide-react'
import { useState } from 'react'
import { Button, cn, Popover, PopoverContent, PopoverTrigger } from 'ui'
-import { TEMPLATES } from '@/components/interfaces/Settings/Logs/Logs.constants'
+import { getLogsTemplates } from '@/components/interfaces/Settings/Logs/Logs.constants'
import type { LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'
import { DefaultLayout } from '@/components/layouts/DefaultLayout'
import LogsLayout from '@/components/layouts/LogsLayout/LogsLayout'
@@ -17,14 +17,16 @@ export const LogsTemplatesPage: NextPageWithLayout = () => {
const { ref: projectRef } = useParams()
const { logsTemplates: isTemplatesEnabled, logsShowMetadataIpTemplate: showMetadataIpTemplate } =
useIsFeatureEnabled(['logs:templates', 'logs:show_metadata_ip_template'])
+ const useOtelEndpoint = useFlag('otelLegacyLogs')
if (!isTemplatesEnabled) {
return <UnknownInterface urlBack={`/project/${projectRef}/logs/explorer`} />
}
+ const templates = getLogsTemplates(useOtelEndpoint)
const allTemplates = showMetadataIpTemplate
- ? TEMPLATES
- : TEMPLATES.filter((template) => template.label !== 'Metadata IP')
+ ? templates
+ : templates.filter((template) => template.label !== 'Metadata IP')
return (
<div className="mx-auto h-full w-full px-5 py-6">