The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(studio): load ClickHouse query templates when OTEL logs are on (#47390)
Jordi Enricb9f76d95f796
diff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.tsindex ec5fc44dbaf..ab6671faf79 100644--- a/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts+++ b/apps/studio/components/interfaces/Settings/Logs/Logs.constants.ts@@ -286,6 +286,157 @@ limit 100;},]+// ClickHouse rewrites of the custom BigQuery templates above, used when the OTEL+// logs engine is on. Keyed by template label. Simple-mode templates are plain+// event_message searches that work unchanged, so they have no entry here.+const OTEL_TEMPLATE_SEARCH_STRINGS: Record<string, string> = {+ 'Commits By User': `select+ log_attributes['parsed.user_name'] as user_name,+ count() as count+from logs+where source = 'postgres_logs'+ and event_message like '%COMMIT%'+group by user_name+order by count desc+limit 100`,+ 'Metadata IP': `select+ timestamp,+ log_attributes['request.headers.x_real_ip'] as x_real_ip+from logs+where source = 'edge_logs'+ and log_attributes['request.headers.x_real_ip'] != ''+order by timestamp desc+limit 100`,+ 'Requests by Geography': `select+ log_attributes['request.cf.country'] as country,+ count() as count+from logs+where source = 'edge_logs'+group by country+order by count desc+limit 100`,+ 'Slow Response Time': `select+ timestamp,+ event_message,+ toInt32OrZero(log_attributes['response.origin_time']) as origin_time+from logs+where source = 'edge_logs'+ and toInt32OrZero(log_attributes['response.origin_time']) > 1000+order by timestamp desc+limit 100`,+ '500 Request Codes': `select+ timestamp,+ event_message,+ toInt32OrZero(log_attributes['response.status_code']) as status_code+from logs+where source = 'edge_logs'+ and toInt32OrZero(log_attributes['response.status_code']) >= 500+order by timestamp desc+limit 100`,+ 'Top Paths': `select+ log_attributes['request.path'] as path,+ log_attributes['request.search'] as params,+ count() as c+from logs+where source = 'edge_logs'+group by path, params+order by c desc+limit 100`,+ 'REST Requests': `select+ timestamp,+ event_message+from logs+where source = 'edge_logs'+ and log_attributes['request.path'] like '%rest/v1%'+order by timestamp desc+limit 100`,+ Errors: `select+ timestamp,+ log_attributes['parsed.error_severity'] as error_severity,+ event_message+from logs+where source = 'postgres_logs'+ and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')+order by timestamp desc+limit 100`,+ 'Error Count by User': `select+ count() as count,+ log_attributes['parsed.user_name'] as user_name,+ log_attributes['parsed.error_severity'] as error_severity+from logs+where source = 'postgres_logs'+ and log_attributes['parsed.error_severity'] in ('ERROR', 'FATAL', 'PANIC')+group by user_name, error_severity+order by count desc+limit 100`,+ 'Auth Endpoint Events': `select+ timestamp,+ event_message+from logs+where source = 'auth_logs'+ and match(event_message, 'level.{3}(info|warning|error|fatal)')+order by timestamp desc+limit 100`,+ 'Auth Audit Logs': `select+ timestamp,+ event_message,+ log_attributes+from logs+where source = 'auth_audit_logs'+order by timestamp desc+limit 10`,+ 'Storage Object Requests': `select+ log_attributes['request.method'] as http_verb,+ log_attributes['request.path'] as filepath,+ count() as num_requests+from logs+where source = 'edge_logs'+ and log_attributes['request.path'] like '%storage/v1/object/%'+group by http_verb, filepath+order by num_requests desc+limit 100`,+ 'Storage Egress Requests': `select+ log_attributes['request.method'] as http_verb,+ log_attributes['request.path'] as filepath,+ (log_attributes['response.headers.cf_cache_status'] = 'HIT') as cached,+ count() as num_requests+from logs+where source = 'edge_logs'+ and (+ log_attributes['request.path'] like '%storage/v1/object/%'+ or log_attributes['request.path'] like '%storage/v1/render/%'+ )+ and log_attributes['request.method'] = 'GET'+group by http_verb, filepath, cached+order by num_requests desc+limit 100`,+ 'Storage Top Cache Misses': `select+ log_attributes['request.path'] as path,+ log_attributes['request.search'] as search,+ count() as count+from logs+where source = 'edge_logs'+ and startsWith(log_attributes['request.path'], '/storage/v1/object')+ and log_attributes['request.method'] = 'GET'+ and log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC')+group by path, search+order by count desc+limit 100`,+}++/**+ * Returns the log templates for the active engine. On the OTEL/ClickHouse engine+ * the custom templates, written for BigQuery, are swapped for their ClickHouse+ * rewrites; everything else is returned unchanged.+ */+export function getLogsTemplates(useOtel: boolean): LogTemplate[] {+ if (!useOtel) return TEMPLATES+ return TEMPLATES.map((template) => {+ const otelSearchString = template.label && OTEL_TEMPLATE_SEARCH_STRINGS[template.label]+ return otelSearchString ? { ...template, searchString: otelSearchString } : template+ })+}+type SqlFilterFn = (value: any) => SafeLogSqlFragmentexport type SqlFilterEntry = SafeLogSqlFragment | SqlFilterFndiff --git a/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.tsnew file mode 100644index 00000000000..f0ee03d2768--- /dev/null+++ b/apps/studio/components/interfaces/Settings/Logs/Logs.templates.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from 'vitest'++import { getLogsTemplates, TEMPLATES } from './Logs.constants'++const customTemplates = TEMPLATES.filter((t) => t.mode === 'custom')+const bigQueryConstructs =+ /unnest|cross join|left join|regexp_contains|cast\(\s*timestamp|count\(\*\)/i++describe('getLogsTemplates', () => {+ it('returns the original templates when the OTEL engine is off', () => {+ expect(getLogsTemplates(false)).toBe(TEMPLATES)+ })++ it('gives every custom template a ClickHouse rewrite when OTEL is on', () => {+ const otel = getLogsTemplates(true)+ for (const template of customTemplates) {+ const rewritten = otel.find((t) => t.label === template.label)+ expect(rewritten, `missing OTEL rewrite for "${template.label}"`).toBeDefined()+ expect(+ rewritten!.searchString,+ `"${template.label}" was not rewritten for ClickHouse`+ ).not.toBe(template.searchString)+ }+ })++ it('produces ClickHouse SQL with no BigQuery constructs', () => {+ const otel = getLogsTemplates(true)+ for (const template of otel.filter((t) => t.mode === 'custom')) {+ expect(template.searchString, `"${template.label}" still targets logs table`).toMatch(+ /from logs/i+ )+ expect(template.searchString, `"${template.label}" filters by source`).toMatch(/source\s*=/i)+ expect(+ template.searchString,+ `"${template.label}" still contains a BigQuery construct`+ ).not.toMatch(bigQueryConstructs)+ }+ })++ it('leaves simple-mode templates unchanged on the OTEL engine', () => {+ const otel = getLogsTemplates(true)+ for (const template of TEMPLATES.filter((t) => t.mode === 'simple')) {+ const same = otel.find((t) => t.label === template.label)+ expect(same!.searchString).toBe(template.searchString)+ }+ })++ it('maps nested BigQuery fields to the right log_attributes keys', () => {+ const otel = getLogsTemplates(true)+ const byLabel = (label: string) => otel.find((t) => t.label === label)!.searchString++ expect(byLabel('Requests by Geography')).toContain("log_attributes['request.cf.country']")+ expect(byLabel('Metadata IP')).toContain("log_attributes['request.headers.x_real_ip']")+ expect(byLabel('Errors')).toContain("log_attributes['parsed.error_severity']")+ expect(byLabel('Slow Response Time')).toContain(+ "toInt32OrZero(log_attributes['response.origin_time'])"+ )+ })+})diff --git a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx b/apps/studio/pages/project/[ref]/logs/explorer/index.tsxindex 1fb54164151..c246e2da385 100644--- a/apps/studio/pages/project/[ref]/logs/explorer/index.tsx+++ b/apps/studio/pages/project/[ref]/logs/explorer/index.tsx@@ -16,8 +16,8 @@ import {import {EXPLORER_DATEPICKER_HELPERS,getDefaultHelper,+ getLogsTemplates,LOGS_LARGE_DATE_RANGE_DAYS_THRESHOLD,- TEMPLATES,} from '@/components/interfaces/Settings/Logs/Logs.constants'import { DatePickerValue } from '@/components/interfaces/Settings/Logs/Logs.DatePickers'import { LogData, LogsWarning, LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'@@ -85,11 +85,13 @@ export const LogsExplorerPage: NextPageWithLayout = () => {const track = useTrack()const projectRef = ref as stringconst { logsShowMetadataIpTemplate } = useIsFeatureEnabled(['logs:show_metadata_ip_template'])+ const useOtelEndpoint = useFlag('otelLegacyLogs')const allTemplates = useMemo(() => {- if (logsShowMetadataIpTemplate) return TEMPLATES- else return TEMPLATES.filter((x) => x.label !== 'Metadata IP')- }, [logsShowMetadataIpTemplate])+ const templates = getLogsTemplates(useOtelEndpoint)+ if (logsShowMetadataIpTemplate) return templates+ else return templates.filter((x) => x.label !== 'Metadata IP')+ }, [logsShowMetadataIpTemplate, useOtelEndpoint])const editorRef = useRef<editor.IStandaloneCodeEditor>(null)const [editorId] = useState<string>(uuidv4())@@ -111,8 +113,6 @@ export const LogsExplorerPage: NextPageWithLayout = () => {}, [timestampStart, timestampEnd, defaultHelper])const [datePickerValue, setDatePickerValue] = useState<DatePickerValue>(initialDatePickerValue)- const useOtelEndpoint = useFlag('otelLegacyLogs')-const { logsDefaultQuery } = useCustomContent(['logs:default_query'])const PLACEHOLDER_QUERY = useOtelEndpoint? OTEL_PLACEHOLDER_QUERYdiff --git a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsxindex d0f507ed041..d5af19ee7cd 100644--- a/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx+++ b/apps/studio/pages/project/[ref]/logs/explorer/templates.tsx@@ -1,9 +1,9 @@-import { useParams } from 'common'+import { useFlag, useParams } from 'common'import { CodeIcon } from 'lucide-react'import { useState } from 'react'import { Button, cn, Popover, PopoverContent, PopoverTrigger } from 'ui'-import { TEMPLATES } from '@/components/interfaces/Settings/Logs/Logs.constants'+import { getLogsTemplates } from '@/components/interfaces/Settings/Logs/Logs.constants'import type { LogTemplate } from '@/components/interfaces/Settings/Logs/Logs.types'import { DefaultLayout } from '@/components/layouts/DefaultLayout'import LogsLayout from '@/components/layouts/LogsLayout/LogsLayout'@@ -17,14 +17,16 @@ export const LogsTemplatesPage: NextPageWithLayout = () => {const { ref: projectRef } = useParams()const { logsTemplates: isTemplatesEnabled, logsShowMetadataIpTemplate: showMetadataIpTemplate } =useIsFeatureEnabled(['logs:templates', 'logs:show_metadata_ip_template'])+ const useOtelEndpoint = useFlag('otelLegacyLogs')if (!isTemplatesEnabled) {return <UnknownInterface urlBack={`/project/${projectRef}/logs/explorer`} />}+ const templates = getLogsTemplates(useOtelEndpoint)const allTemplates = showMetadataIpTemplate- ? TEMPLATES- : TEMPLATES.filter((template) => template.label !== 'Metadata IP')+ ? templates+ : templates.filter((template) => template.label !== 'Metadata IP')return (<div className="mx-auto h-full w-full px-5 py-6">