Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(functions): migrate last-hour stats query to OTEL ClickHouse (#47693)

Jordi Enric982d86012340
diff --git a/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.test.ts b/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.test.ts
index 9ab03297e99..c94fc2c62f9 100644
--- a/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.test.ts
+++ b/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.test.ts
@@ -21,7 +21,7 @@ describe('getEdgeFunctionsLastHourStats', () => {
vi.useRealTimers()
})
- it('requests last-hour function stats from logs.all', async () => {
+ it('requests last-hour function stats from logs.all by default', async () => {
vi.mocked(post).mockResolvedValue({ data: { result: [] }, error: null } as PostResponse)
await getEdgeFunctionsLastHourStats({
@@ -49,6 +49,42 @@ describe('getEdgeFunctionsLastHourStats', () => {
expect(postCalls[0]?.[1]?.body?.sql).toContain('from\n function_edge_logs')
})
+ it('requests last-hour function stats from logs.all.otel when useOtel is set', async () => {
+ vi.mocked(post).mockResolvedValue({ data: { result: [] }, error: null } as PostResponse)
+
+ await getEdgeFunctionsLastHourStats({
+ projectRef: 'project-ref',
+ functionIds: ['fn_1', 'fn_2'],
+ useOtel: true,
+ })
+
+ expect(post).toHaveBeenCalledWith(
+ '/platform/projects/{ref}/analytics/endpoints/logs.all.otel',
+ {
+ params: {
+ path: { ref: 'project-ref' },
+ query: { key: 'last-hour-stats' },
+ },
+ body: expect.objectContaining({
+ sql: expect.stringContaining(`and log_attributes['function_id'] in ('fn_1', 'fn_2')`),
+ iso_timestamp_start: '2024-01-15T11:00:00.000Z',
+ iso_timestamp_end: '2024-01-15T12:00:00.000Z',
+ }),
+ signal: undefined,
+ }
+ )
+
+ const postCalls = vi.mocked(post).mock.calls as Array<
+ [string, { body?: { sql?: string } } | undefined]
+ >
+ const sql = postCalls[0]?.[1]?.body?.sql ?? ''
+
+ expect(sql).toContain('from logs')
+ expect(sql).toContain("source = 'function_edge_logs'")
+ expect(sql).toContain("case when toInt32OrZero(log_attributes['response.status_code']) >= 500")
+ expect(sql).not.toContain('cross join unnest')
+ })
+
it('coerces counts to numbers and computes error rates per function', async () => {
vi.mocked(post).mockResolvedValue({
data: {
diff --git a/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.ts b/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.ts
index a1b6305bb7d..9f372c66901 100644
--- a/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.ts
+++ b/apps/studio/data/edge-functions/edge-functions-last-hour-stats-query.ts
@@ -1,9 +1,11 @@
import { useQuery } from '@tanstack/react-query'
+import { useFlag } from 'common'
import dayjs from 'dayjs'
import { edgeFunctionsKeys } from './keys'
import { handleError } from '@/data/fetchers'
import { executeAnalyticsSql } from '@/data/logs/execute-analytics-sql'
+import { logsAllEndpointUrl } from '@/data/logs/logs-endpoint'
import {
analyticsLiteral,
joinSqlFragments,
@@ -12,7 +14,11 @@ import {
} from '@/data/logs/safe-analytics-sql'
import type { ResponseError, UseCustomQueryOptions } from '@/types'
-export type EdgeFunctionsLastHourStatsVariables = { projectRef?: string; functionIds?: string[] }
+export type EdgeFunctionsLastHourStatsVariables = {
+ projectRef?: string
+ functionIds?: string[]
+ useOtel?: boolean
+}
export type EdgeFunctionLastHourStats = {
functionId: string
@@ -23,7 +29,7 @@ export type EdgeFunctionLastHourStats = {
export type EdgeFunctionsLastHourStatsResponse = Record<string, EdgeFunctionLastHourStats>
-function getEdgeFunctionsLastHourStatsSql(functionIds: string[]): SafeLogSqlFragment {
+function getEdgeFunctionsLastHourStatsSqlBq(functionIds: string[]): SafeLogSqlFragment {
const functionIdFilter: SafeLogSqlFragment =
functionIds.length > 0
? safeSql` and function_id in (${joinSqlFragments(functionIds.map(analyticsLiteral), ', ')})\n`
@@ -46,8 +52,38 @@ ${functionIdFilter}group by
`
}
+function getEdgeFunctionsLastHourStatsSqlOtel(functionIds: string[]): SafeLogSqlFragment {
+ const functionIdFilter: SafeLogSqlFragment =
+ functionIds.length > 0
+ ? safeSql` and log_attributes['function_id'] in (${joinSqlFragments(functionIds.map(analyticsLiteral), ', ')})\n`
+ : safeSql``
+
+ return safeSql`
+-- edge-functions-last-hour-stats
+select
+ log_attributes['function_id'] as function_id,
+ count(distinct id) as requests_count,
+ count(distinct case when toInt32OrZero(log_attributes['response.status_code']) >= 500 then id end) as server_err_count
+from logs
+where
+ source = 'function_edge_logs'
+ and log_attributes['function_id'] != ''
+${functionIdFilter}group by
+ function_id
+`
+}
+
+function getEdgeFunctionsLastHourStatsSql(
+ functionIds: string[],
+ useOtel: boolean
+): SafeLogSqlFragment {
+ return useOtel
+ ? getEdgeFunctionsLastHourStatsSqlOtel(functionIds)
+ : getEdgeFunctionsLastHourStatsSqlBq(functionIds)
+}
+
export async function getEdgeFunctionsLastHourStats(
- { projectRef, functionIds = [] }: EdgeFunctionsLastHourStatsVariables,
+ { projectRef, functionIds = [], useOtel = false }: EdgeFunctionsLastHourStatsVariables,
signal?: AbortSignal
) {
if (!projectRef) throw new Error('projectRef is required')
@@ -58,8 +94,8 @@ export async function getEdgeFunctionsLastHourStats(
const data = await executeAnalyticsSql({
projectRef,
- endpoint: '/platform/projects/{ref}/analytics/endpoints/logs.all',
- sql: getEdgeFunctionsLastHourStatsSql(functionIds),
+ endpoint: logsAllEndpointUrl(useOtel),
+ sql: getEdgeFunctionsLastHourStatsSql(functionIds, useOtel),
iso_timestamp_start: startDate,
iso_timestamp_end: endDate,
key: 'last-hour-stats',
@@ -108,12 +144,16 @@ export const useEdgeFunctionsLastHourStatsQuery = <TData = EdgeFunctionsLastHour
EdgeFunctionsLastHourStatsError,
TData
> = {}
-) =>
- useQuery<EdgeFunctionsLastHourStatsData, EdgeFunctionsLastHourStatsError, TData>({
- queryKey: edgeFunctionsKeys.lastHourStats(projectRef, functionIds),
- queryFn: ({ signal }) => getEdgeFunctionsLastHourStats({ projectRef, functionIds }, signal),
+) => {
+ const useOtel = useFlag('otelLegacyLogs')
+
+ return useQuery<EdgeFunctionsLastHourStatsData, EdgeFunctionsLastHourStatsError, TData>({
+ queryKey: edgeFunctionsKeys.lastHourStats(projectRef, functionIds, useOtel),
+ queryFn: ({ signal }) =>
+ getEdgeFunctionsLastHourStats({ projectRef, functionIds, useOtel }, signal),
enabled: enabled && typeof projectRef !== 'undefined' && functionIds.length > 0,
staleTime: 60 * 1000,
retry: false,
...options,
})
+}
diff --git a/apps/studio/data/edge-functions/keys.ts b/apps/studio/data/edge-functions/keys.ts
index 1b13ea0eba7..11aaa18993e 100644
--- a/apps/studio/data/edge-functions/keys.ts
+++ b/apps/studio/data/edge-functions/keys.ts
@@ -3,13 +3,18 @@ export const normalizeFunctionIds = (functionIds: string[]): string[] =>
export const edgeFunctionsKeys = {
list: (projectRef: string | undefined) => ['projects', projectRef, 'edge-functions'] as const,
- lastHourStats: (projectRef: string | undefined, functionIds: string[] = []) =>
+ lastHourStats: (
+ projectRef: string | undefined,
+ functionIds: string[] = [],
+ useOtel: boolean = false
+ ) =>
[
'projects',
projectRef,
'edge-functions',
'last-hour-stats',
normalizeFunctionIds(functionIds),
+ { otel: useOtel },
] as const,
detail: (projectRef: string | undefined, slug: string | undefined) =>
['projects', projectRef, 'edge-function', slug, 'detail'] as const,