The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(studio): migrate Storage report to OTEL (#50519)
Jordi Enric7d3e8fa5a356
diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.otel.test.ts b/apps/studio/components/interfaces/Reports/Reports.constants.otel.test.tsnew file mode 100644index 00000000000..34e6f2857dd--- /dev/null+++ b/apps/studio/components/interfaces/Reports/Reports.constants.otel.test.ts@@ -0,0 +1,128 @@+import { describe, expect, it } from 'vitest'++import { generateOtelWhereSafe, PRESET_CONFIG } from './Reports.constants'+import { Presets, type ReportFilterItem } from './Reports.types'++function getOtelSql(preset: Presets, queryName: string, filters: ReportFilterItem[] = []): string {+ const query = PRESET_CONFIG[preset].queries[queryName]+ if (query.queryType !== 'logs' || query.safeSqlOtel === undefined) {+ throw new Error(`Missing OTEL logs query for ${preset}.${queryName}`)+ }+ return query.safeSqlOtel(filters)+}++describe('generateOtelWhereSafe', () => {+ it.each([+ ['matches', 'api.*', "match(log_attributes['request.path'], 'api.*')"],+ ['is', 'GET', "log_attributes['request.path'] = 'GET'"],+ ['!=', 'GET', "log_attributes['request.path'] != 'GET'"],+ ['>=', '400', "toFloat64OrNull(log_attributes['request.path']) >= 400"],+ ['<=', '400', "toFloat64OrNull(log_attributes['request.path']) <= 400"],+ ['>', '400', "toFloat64OrNull(log_attributes['request.path']) > 400"],+ ['<', '400', "toFloat64OrNull(log_attributes['request.path']) < 400"],+ ] as const)('builds a safe %s predicate', (compare, value, expected) => {+ expect(generateOtelWhereSafe([{ key: 'request.path', value, compare }])).toBe(+ `WHERE ${expected}`+ )+ })++ it('preserves dotted keys and escapes key and value input', () => {+ expect(+ generateOtelWhereSafe([+ {+ key: "request.headers.user_agent'] OR 1=1 --",+ value: "agent' OR 1=1 --",+ compare: 'is',+ },+ ])+ ).toBe("WHERE log_attributes['request.headers.user_agent''] OR 1=1 --'] = 'agent'' OR 1=1 --'")+ })++ it('omits invalid numeric comparisons without dropping valid filters', () => {+ expect(+ generateOtelWhereSafe(+ [+ { key: 'response.status_code', value: 'not-a-number', compare: '>=' },+ { key: 'request.method', value: 'GET', compare: 'is' },+ ],+ false+ )+ ).toBe("AND log_attributes['request.method'] = 'GET'")+ })+})++describe('Storage report OTEL queries', () => {+ it.each([+ ['totalRequests', 50000],+ ['topRoutes', 10],+ ['errorCounts', 50000],+ ['topErrorRoutes', 10],+ ['responseSpeed', 50000],+ ['topSlowRoutes', 10],+ ['networkTraffic', 50000],+ ])('uses edge_logs and a fixed limit for API query %s', (queryName, limit) => {+ const sql = getOtelSql(Presets.API, queryName)++ expect(sql).toContain('from logs')+ expect(sql).toContain("source = 'edge_logs'")+ expect(sql).toContain(`limit ${limit}`)+ expect(sql).not.toContain('from edge_logs')+ expect(sql).not.toContain('cross join unnest')+ })++ it.each(['totalRequests', 'errorCounts'])(+ 'emits numeric counts and explicit unix-microsecond buckets for %s',+ (queryName) => {+ const sql = getOtelSql(Presets.API, queryName)+ const bucket = 'toUnixTimestamp(toStartOfHour(logs.timestamp)) * 1000000'++ expect(sql).toContain(`${bucket} as timestamp`)+ expect(sql).toContain('toFloat64(count()) as count')+ expect(sql).toContain(`group by ${bucket}`)+ expect(sql).toContain(`order by ${bucket} asc`)+ }+ )++ it('uses exact route attributes and numeric response values', () => {+ const routesSql = getOtelSql(Presets.API, 'topRoutes')+ const speedSql = getOtelSql(Presets.API, 'responseSpeed')+ const trafficSql = getOtelSql(Presets.API, 'networkTraffic')++ expect(routesSql).toContain("log_attributes['request.path'] as path")+ expect(routesSql).toContain("log_attributes['request.method'] as method")+ expect(routesSql).toContain("log_attributes['request.search'] as search")+ expect(routesSql).toContain(+ "toInt32OrZero(log_attributes['response.status_code']) as status_code"+ )+ expect(routesSql).toContain('toFloat64(count()) as count')+ expect(speedSql).toContain(+ "avg(toFloat64OrNull(log_attributes['response.origin_time'])) as avg"+ )+ expect(trafficSql).toContain(+ "sum(toFloat64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb"+ )+ expect(trafficSql).toContain(+ "sum(toFloat64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb"+ )+ })++ it('builds numeric Storage cache results from exact OTEL attributes', () => {+ const hitRateSql = getOtelSql(Presets.STORAGE, 'cacheHitRate')+ const missesSql = getOtelSql(Presets.STORAGE, 'topCacheMisses')++ expect(hitRateSql).toContain("where source = 'edge_logs'")+ expect(hitRateSql).toContain("startsWith(log_attributes['request.path'], '/storage/v1/object')")+ expect(hitRateSql).toContain(+ "toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ('HIT', 'STALE', 'REVALIDATED', 'UPDATING'))) as hit_count"+ )+ expect(hitRateSql).toContain(+ "toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC'))) as miss_count"+ )+ expect(hitRateSql).toContain('limit 50000')+ expect(missesSql).toContain('toFloat64(count()) as count')+ expect(missesSql).toContain(+ "log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC')"+ )+ expect(missesSql).toContain('limit 12')+ })+})diff --git a/apps/studio/components/interfaces/Reports/Reports.constants.ts b/apps/studio/components/interfaces/Reports/Reports.constants.tsindex 277ccb91d02..2bea60fc78a 100644--- a/apps/studio/components/interfaces/Reports/Reports.constants.ts+++ b/apps/studio/components/interfaces/Reports/Reports.constants.ts@@ -143,6 +143,83 @@ export function generateRegexpWhereSafe(return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}`}+export function generateOtelWhereSafe(+ filters: ReportFilterItem[],+ prepend = true+): SafeLogSqlFragment {+ const conditions = filters+ .map((filter) => {+ const column = safeLogSql`log_attributes[${analyticsLiteral(filter.key)}]`+ const stringValue = analyticsLiteral(String(filter.value))++ switch (filter.compare) {+ case 'matches':+ return safeLogSql`match(${column}, ${stringValue})`+ case 'is':+ return safeLogSql`${column} = ${stringValue}`+ case '!=':+ return safeLogSql`${column} != ${stringValue}`+ case '>=':+ case '<=':+ case '>':+ case '<': {+ const numericValue =+ typeof filter.value === 'number' ? filter.value : Number(filter.value.trim())+ if (String(filter.value).trim() === '' || !Number.isFinite(numericValue)) return null++ const numericColumn = safeLogSql`toFloat64OrNull(${column})`+ const literalValue = analyticsLiteral(numericValue)+ if (filter.compare === '>=') return safeLogSql`${numericColumn} >= ${literalValue}`+ if (filter.compare === '<=') return safeLogSql`${numericColumn} <= ${literalValue}`+ if (filter.compare === '>') return safeLogSql`${numericColumn} > ${literalValue}`+ return safeLogSql`${numericColumn} < ${literalValue}`+ }+ }+ })+ .filter((condition): condition is SafeLogSqlFragment => condition !== null)++ if (conditions.length === 0) return safeLogSql``++ const joined = joinSqlFragments(conditions, ' AND ')+ return prepend ? safeLogSql`WHERE ${joined}` : safeLogSql`AND ${joined}`+}++const OTEL_HOURLY_TIMESTAMP = safeLogSql`toUnixTimestamp(toStartOfHour(logs.timestamp)) * 1000000`+const OTEL_STATUS_CODE = safeLogSql`toInt32OrZero(log_attributes['response.status_code'])`+const OTEL_ORIGIN_TIME = safeLogSql`toFloat64OrNull(log_attributes['response.origin_time'])`+const OTEL_ROUTE_SELECT = safeLogSql`+ log_attributes['request.path'] as path,+ log_attributes['request.method'] as method,+ log_attributes['request.search'] as search,+ ${OTEL_STATUS_CODE} as status_code`+const OTEL_ROUTE_GROUP_BY = safeLogSql`+ log_attributes['request.path'],+ log_attributes['request.method'],+ log_attributes['request.search'],+ ${OTEL_STATUS_CODE}`+const OTEL_ERROR_STATUS = safeLogSql`${OTEL_STATUS_CODE} >= 400`++function otelWhere(filters: ReportFilterItem[], extra?: SafeLogSqlFragment): SafeLogSqlFragment {+ const base = extra+ ? safeLogSql`where source = 'edge_logs' and ${extra}`+ : safeLogSql`where source = 'edge_logs'`+ const filterSql = generateOtelWhereSafe(filters, false)+ return filterSql.length > 0 ? safeLogSql`${base} ${filterSql}` : base+}++function statusList(statuses: string[]): SafeLogSqlFragment {+ return safeLogSql`(${joinSqlFragments(statuses.map(analyticsLiteral), ', ')})`+}++const STORAGE_CACHE_HIT_STATUSES = statusList(['HIT', 'STALE', 'REVALIDATED', 'UPDATING'])+const STORAGE_CACHE_MISS_STATUSES = statusList([+ 'MISS',+ 'NONE/UNKNOWN',+ 'EXPIRED',+ 'BYPASS',+ 'DYNAMIC',+])+export const PRESET_CONFIG: Record<Presets, PresetConfig> = {[Presets.API]: {title: 'API',@@ -164,6 +241,15 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {timestampORDER BYtimestamp ASC`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_HOURLY_TIMESTAMP} as timestamp,+ toFloat64(count()) as count+ from logs+ ${otelWhere(filters)}+ group by ${OTEL_HOURLY_TIMESTAMP}+ order by ${OTEL_HOURLY_TIMESTAMP} asc+ limit 50000`,},topRoutes: {queryType: 'logs',@@ -187,6 +273,15 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {count desclimit 10`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_ROUTE_SELECT},+ toFloat64(count()) as count+ from logs+ ${otelWhere(filters)}+ group by ${OTEL_ROUTE_GROUP_BY}+ order by count desc+ limit 10`,},errorCounts: {queryType: 'logs',@@ -208,6 +303,15 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {ORDER BYtimestamp ASC`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_HOURLY_TIMESTAMP} as timestamp,+ toFloat64(count()) as count+ from logs+ ${otelWhere(filters, OTEL_ERROR_STATUS)}+ group by ${OTEL_HOURLY_TIMESTAMP}+ order by ${OTEL_HOURLY_TIMESTAMP} asc+ limit 50000`,},topErrorRoutes: {queryType: 'logs',@@ -233,6 +337,15 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {count desclimit 10`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_ROUTE_SELECT},+ toFloat64(count()) as count+ from logs+ ${otelWhere(filters, OTEL_ERROR_STATUS)}+ group by ${OTEL_ROUTE_GROUP_BY}+ order by count desc+ limit 10`,},responseSpeed: {queryType: 'logs',@@ -253,6 +366,15 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {ORDER BYtimestamp ASC`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_HOURLY_TIMESTAMP} as timestamp,+ avg(${OTEL_ORIGIN_TIME}) as avg+ from logs+ ${otelWhere(filters)}+ group by ${OTEL_HOURLY_TIMESTAMP}+ order by ${OTEL_HOURLY_TIMESTAMP} asc+ limit 50000`,},topSlowRoutes: {queryType: 'logs',@@ -277,6 +399,16 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {avg desclimit 10`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_ROUTE_SELECT},+ toFloat64(count()) as count,+ avg(${OTEL_ORIGIN_TIME}) as avg+ from logs+ ${otelWhere(filters)}+ group by ${OTEL_ROUTE_GROUP_BY}+ order by avg desc+ limit 10`,},networkTraffic: {queryType: 'logs',@@ -315,6 +447,16 @@ export const PRESET_CONFIG: Record<Presets, PresetConfig> = {ORDER BYtimestamp ASC`,+ safeSqlOtel: (filters) => safeLogSql`+ select+ ${OTEL_HOURLY_TIMESTAMP} as timestamp,+ sum(toFloat64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb,+ sum(toFloat64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb+ from logs+ ${otelWhere(filters)}+ group by ${OTEL_HOURLY_TIMESTAMP}+ order by ${OTEL_HOURLY_TIMESTAMP} asc+ limit 50000`,},requestsByCountry: {queryType: 'logs',@@ -364,6 +506,19 @@ where starts_with(r.path, '/storage/v1/object') and r.method = 'GET'group by timestamporder by timestamp desc`,+ safeSqlOtel: (filters) => safeLogSql`+select+ ${OTEL_HOURLY_TIMESTAMP} as timestamp,+ toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ${STORAGE_CACHE_HIT_STATUSES})) as hit_count,+ toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ${STORAGE_CACHE_MISS_STATUSES})) as miss_count+from logs+where source = 'edge_logs'+ and startsWith(log_attributes['request.path'], '/storage/v1/object')+ and log_attributes['request.method'] = 'GET'+ ${generateOtelWhereSafe(filters, false)}+group by ${OTEL_HOURLY_TIMESTAMP}+order by ${OTEL_HOURLY_TIMESTAMP} desc+limit 50000`,},topCacheMisses: {queryType: 'logs',@@ -387,6 +542,20 @@ group by path, searchorder by count desclimit 12`,+ safeSqlOtel: (filters) => safeLogSql`+select+ log_attributes['request.path'] as path,+ log_attributes['request.search'] as search,+ toFloat64(count()) as count+from logs+where source = 'edge_logs'+ and startsWith(log_attributes['request.path'], '/storage/v1/object')+ and log_attributes['request.method'] = 'GET'+ and log_attributes['response.headers.cf_cache_status'] in ${STORAGE_CACHE_MISS_STATUSES}+ ${generateOtelWhereSafe(filters, false)}+group by log_attributes['request.path'], log_attributes['request.search']+order by count desc+limit 12`,},},},diff --git a/apps/studio/components/interfaces/Reports/Reports.types.ts b/apps/studio/components/interfaces/Reports/Reports.types.tsindex f4c510296ab..9892f47383b 100644--- a/apps/studio/components/interfaces/Reports/Reports.types.ts+++ b/apps/studio/components/interfaces/Reports/Reports.types.ts@@ -33,6 +33,7 @@ export interface ReportQueryLogs {page?: number,pageSize?: number) => SafeLogSqlFragment+ safeSqlOtel?: (filters: ReportFilterItem[]) => SafeLogSqlFragment}export interface ReportQueryDb {diff --git a/apps/studio/components/interfaces/Reports/Reports.utils.tsx b/apps/studio/components/interfaces/Reports/Reports.utils.tsxindex e844f6241db..12323457240 100644--- a/apps/studio/components/interfaces/Reports/Reports.utils.tsx+++ b/apps/studio/components/interfaces/Reports/Reports.utils.tsx@@ -47,7 +47,9 @@ type PresetHooks = Record<keyof PresetConfig['queries'], () => PresetHookResult>*/export const queriesFactory = <T extends string>(queries: BaseQueries<T>,- projectRef: string+ projectRef?: string,+ useOtel = false,+ enabled = true): PresetHooks => {const hooks: PresetHooks = Object.entries<ReportQuery>(queries).reduce((acc, [k, query]) => {if (query.queryType === 'db') {@@ -58,17 +60,27 @@ export const queriesFactory = <T extends string>(} else {return {...acc,- [k]: () => useLogsQuery({ projectRef }),+ [k]: () => useLogsQuery({ projectRef, enabled, options: { useOtel } }),}}}, {})return hooks}-export function getLogsSql(query: ReportQuery, filters: ReportFilterItem[]): SafeLogSqlFragment {+export function getLogsSql(+ query: ReportQuery,+ filters: ReportFilterItem[],+ useOtel = false+): SafeLogSqlFragment {if (query.queryType !== 'logs') {throw new Error(`Expected logs query, got ${query.queryType}`)}+ if (useOtel) {+ if (query.safeSqlOtel === undefined) {+ throw new Error('Expected an OTEL logs query')+ }+ return query.safeSqlOtel(filters)+ }return query.safeSql(filters)}diff --git a/apps/studio/data/reports/storage-report-query.ts b/apps/studio/data/reports/storage-report-query.tsindex 56f2432bb34..61eda564f10 100644--- a/apps/studio/data/reports/storage-report-query.ts+++ b/apps/studio/data/reports/storage-report-query.ts@@ -1,27 +1,40 @@-import { useParams } from 'common'+import { FeatureFlagContext, useFlag, useParams } from 'common'import isEqual from 'lodash/isEqual'-import { useEffect, useState } from 'react'+import { useContext, useEffect, useState } from 'react'import { PRESET_CONFIG } from '@/components/interfaces/Reports/Reports.constants'import { ReportFilterItem } from '@/components/interfaces/Reports/Reports.types'import { getLogsSql, queriesFactory } from '@/components/interfaces/Reports/Reports.utils'import type { LogsEndpointParams } from '@/components/interfaces/Settings/Logs/Logs.types'+import { IS_PLATFORM } from '@/lib/constants'import { useDatabaseSelectorStateSnapshot } from '@/state/database-selector'export const useStorageReport = () => {const { ref: projectRef } = useParams()const [filters, setFilters] = useState<ReportFilterItem[]>([])const state = useDatabaseSelectorStateSnapshot()+ const desiredUseOtel = useFlag('otelReports')+ const { hasLoaded: hasLoadedFlags } = useContext(FeatureFlagContext)+ const [appliedUseOtel, setAppliedUseOtel] = useState<boolean | null>(null)const identifier = state.selectedDatabaseId+ const hasAppliedQueryMode = appliedUseOtel !== null+ const useOtel = appliedUseOtel ?? false+ const resolvedProjectRef = IS_PLATFORM ? projectRef : (projectRef ?? 'default')+ const hasResolvedProjectRef = !IS_PLATFORM || projectRef !== undefined+ const areQueriesReady = hasAppliedQueryMode && hasResolvedProjectRefconst queryHooks = queriesFactory<keyof typeof PRESET_CONFIG.api.queries>(PRESET_CONFIG.api.queries,- projectRef ?? 'default'+ resolvedProjectRef,+ useOtel,+ areQueriesReady)const storageQueryHooks = queriesFactory<keyof typeof PRESET_CONFIG.storage.queries>(PRESET_CONFIG.storage.queries,- projectRef ?? 'default'+ resolvedProjectRef,+ useOtel,+ areQueriesReady)const totalRequests = queryHooks.totalRequests()const topRoutes = queryHooks.topRoutes()@@ -45,6 +58,7 @@ export const useStorageReport = () => {]const handleRefresh = async () => {+ if (!areQueriesReady) returnactiveHooks.forEach((hook) => hook.runQuery())}const handleSetParams = (params: Partial<LogsEndpointParams>) => {@@ -85,51 +99,64 @@ export const useStorageReport = () => {]useEffect(() => {+ if (IS_PLATFORM && !hasLoadedFlags) return++ const nextUseOtel = IS_PLATFORM ? desiredUseOtel : false+if (totalRequests.changeQuery) {totalRequests.changeQuery(- getLogsSql(PRESET_CONFIG.api.queries.totalRequests, formattedFilters)+ getLogsSql(PRESET_CONFIG.api.queries.totalRequests, formattedFilters, nextUseOtel))}if (topRoutes.changeQuery) {- topRoutes.changeQuery(getLogsSql(PRESET_CONFIG.api.queries.topRoutes, formattedFilters))+ topRoutes.changeQuery(+ getLogsSql(PRESET_CONFIG.api.queries.topRoutes, formattedFilters, nextUseOtel)+ )}if (errorCounts.changeQuery) {- errorCounts.changeQuery(getLogsSql(PRESET_CONFIG.api.queries.errorCounts, formattedFilters))+ errorCounts.changeQuery(+ getLogsSql(PRESET_CONFIG.api.queries.errorCounts, formattedFilters, nextUseOtel)+ )}if (topErrorRoutes.changeQuery) {topErrorRoutes.changeQuery(- getLogsSql(PRESET_CONFIG.api.queries.topErrorRoutes, formattedFilters)+ getLogsSql(PRESET_CONFIG.api.queries.topErrorRoutes, formattedFilters, nextUseOtel))}if (responseSpeed.changeQuery) {responseSpeed.changeQuery(- getLogsSql(PRESET_CONFIG.api.queries.responseSpeed, formattedFilters)+ getLogsSql(PRESET_CONFIG.api.queries.responseSpeed, formattedFilters, nextUseOtel))}if (topSlowRoutes.changeQuery) {topSlowRoutes.changeQuery(- getLogsSql(PRESET_CONFIG.api.queries.topSlowRoutes, formattedFilters)+ getLogsSql(PRESET_CONFIG.api.queries.topSlowRoutes, formattedFilters, nextUseOtel))}if (networkTraffic.changeQuery) {networkTraffic.changeQuery(- getLogsSql(PRESET_CONFIG.api.queries.networkTraffic, formattedFilters)+ getLogsSql(PRESET_CONFIG.api.queries.networkTraffic, formattedFilters, nextUseOtel))}if (cacheHitRate.changeQuery) {- cacheHitRate.changeQuery(getLogsSql(PRESET_CONFIG.storage.queries.cacheHitRate, []))+ cacheHitRate.changeQuery(+ getLogsSql(PRESET_CONFIG.storage.queries.cacheHitRate, [], nextUseOtel)+ )}if (topCacheMisses.changeQuery) {- topCacheMisses.changeQuery(getLogsSql(PRESET_CONFIG.storage.queries.topCacheMisses, []))+ topCacheMisses.changeQuery(+ getLogsSql(PRESET_CONFIG.storage.queries.topCacheMisses, [], nextUseOtel)+ )}- }, [JSON.stringify(formattedFilters)])+ setAppliedUseOtel(nextUseOtel)+ }, [JSON.stringify(formattedFilters), desiredUseOtel, hasLoadedFlags])- const isLoading = activeHooks.some((hook) => hook.isLoading)+ const isLoading = !areQueriesReady || activeHooks.some((hook) => hook.isLoading)return {data: {