Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

perf(studio): single-scan unified logs facet count query (#47088)

Jordi Enricb87110b695eb
diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts
index 5d9d9f8380f..ba73e93c0b4 100644
--- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts
+++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts
@@ -134,29 +134,40 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {
})
describe('getLogsCountQuery', () => {
- it('emits one UNION ALL branch per log_type bucket and per level', () => {
+ const whereOfBranchContaining = (sql: string, needle: string) => {
+ const branch = sql.split(/\bUNION ALL\b/).find((b) => b.includes(needle)) ?? ''
+ return branch.split(/\bWHERE\b/)[1]?.split(/\bGROUP BY\b/)[0] ?? ''
+ }
+
+ it('folds facets into single-pass arrayJoin scans with a total row', () => {
const sql = getLogsCountQuery(baseSearch)
- // Per-log-type counts
+ expect(sql).toContain('arrayJoin([')
+ expect(sql).toContain('multiIf(')
+ expect(sql).toContain(`facet = 'total', 'all'`)
for (const lt of ['edge', 'postgrest', 'storage', 'postgres', 'edge function', 'auth']) {
expect(sql).toContain(`'${lt}'`)
}
- // Per-level counts
for (const lvl of ['success', 'warning', 'error']) {
expect(sql).toContain(`'${lvl}'`)
}
- // Bundled via UNION ALL — multiple occurrences expected
- expect(sql.match(/UNION ALL/g)?.length ?? 0).toBeGreaterThan(5)
+ expect(sql).toContain(`'pathname'`)
+ expect(sql).toContain('LIMIT 20')
+ // log_type + base + pathname = 3 scans
+ expect(sql.match(/FROM logs/g)?.length ?? 0).toBeLessThanOrEqual(4)
+ })
+
+ it('honours an active log_type filter in the total count scan', () => {
+ const sql = getLogsCountQuery(withFilters('log_type:eq:edge'))
+ // Assert on the WHERE only: value expressions mention other sources inline.
+ const totalWhere = whereOfBranchContaining(sql, `'all'`)
+ expect(totalWhere).toContain(`source = 'edge_logs'`)
+ expect(totalWhere).not.toContain(`source = 'postgres_logs'`)
})
- it('honours an active log_type filter in the total count branch', () => {
+ it('gives the log_type facet its own scan that excludes the log_type filter', () => {
const sql = getLogsCountQuery(withFilters('log_type:eq:edge'))
- // The first branch is the total — its WHERE must include the edge
- // log_type predicate, otherwise the total badge would over-count
- // when a log_type filter is active.
- const totalBranch = sql.split(/\bUNION ALL\b/)[0]
- expect(totalBranch).toContain(`'total'`)
- expect(totalBranch).toContain(`source = 'edge_logs'`)
- expect(totalBranch).not.toContain(`source = 'postgres_logs'`)
+ const logTypeWhere = whereOfBranchContaining(sql, `'log_type'`)
+ expect(logTypeWhere).not.toContain(`NOT LIKE '%/rest/%'`)
})
})
diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts
index 39ab733e62e..30121d41b21 100644
--- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts
+++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts
@@ -45,7 +45,7 @@ const ATTR = {
const HTTP_STATUS_EXPR: SafeLogSqlFragment = safeSql`if(source = 'auth_logs', log_attributes['status'], ${ATTR.status})`
/**
- * Predicate that matches rows belonging to a given log_type. Mirrors the
+ * Condition that matches rows belonging to a given log_type. Mirrors the
* shape of the original BigQuery unified-logs CTEs: edge gateway traffic
* (`source = 'edge_logs'`) is split between `edge`, `postgrest` and `storage`
* based on URL path. Other types map straight to a single source.
@@ -54,7 +54,7 @@ const HTTP_STATUS_EXPR: SafeLogSqlFragment = safeSql`if(source = 'auth_logs', lo
* logs from postgREST / storage-api and are intentionally not part of unified
* logs; the UI surfaces gateway HTTP traffic for those buckets.
*/
-const LOG_TYPE_PREDICATE: Record<string, SafeLogSqlFragment> = {
+const LOG_TYPE_CONDITION: Record<string, SafeLogSqlFragment> = {
edge: safeSql`source = 'edge_logs' AND ${ATTR.path} NOT LIKE '%/rest/%' AND ${ATTR.path} NOT LIKE '%/storage/%'`,
postgrest: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%'`,
storage: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/storage/%'`,
@@ -106,18 +106,18 @@ const LEVEL_EXPR: SafeLogSqlFragment = safeSql`CASE
ELSE 'success'
END`
-const logTypeWherePredicate = (logTypes: string[]): SafeLogSqlFragment => {
- const effective = logTypes.filter((t) => t in LOG_TYPE_PREDICATE)
+const logTypeWhereCondition = (logTypes: string[]): SafeLogSqlFragment => {
+ const effective = logTypes.filter((t) => t in LOG_TYPE_CONDITION)
const types = effective.length ? effective : [...DEFAULT_LOG_TYPES]
- const branches = types.map((t) => safeSql`(${LOG_TYPE_PREDICATE[t]})`)
+ const branches = types.map((t) => safeSql`(${LOG_TYPE_CONDITION[t]})`)
return safeSql`(${joinSqlFragments(branches, ' OR ')})`
}
/**
* Translates one (column, values, operator) group from the parsed `filter` URL
- * param into an underlying SQL predicate. The OTEL endpoint rejects queries
+ * param into an underlying SQL condition. The OTEL endpoint rejects queries
* that reference derived aliases like `log_type` or `level` in WHERE for some
- * shapes, so we always emit raw-column predicates (source/severity_text/
+ * shapes, so we always emit raw-column conditions (source/severity_text/
* log_attributes[…]). For `<>`, IN becomes NOT IN, LIKE becomes NOT LIKE, and
* multi-value lists are joined with AND so the row must not match *any* value.
*/
@@ -142,8 +142,8 @@ const translateFilter = (
switch (key) {
case 'log_type': {
const branches = values.map((t) => {
- const pred = LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`
- return isNeq ? safeSql`NOT (${pred})` : safeSql`(${pred})`
+ const condition = LOG_TYPE_CONDITION[t] ?? safeSql`source = ${lit(t)}`
+ return isNeq ? safeSql`NOT (${condition})` : safeSql`(${condition})`
})
return safeSql`(${joinSqlFragments(branches, joinAndOr)})`
}
@@ -193,8 +193,8 @@ const translateFilter = (
}
}
-const whereClause = (predicates: SafeLogSqlFragment[]): SafeLogSqlFragment =>
- predicates.length > 0 ? safeSql`WHERE ${joinSqlFragments(predicates, ' AND ')}` : safeSql``
+const whereClause = (conditions: SafeLogSqlFragment[]): SafeLogSqlFragment =>
+ conditions.length > 0 ? safeSql`WHERE ${joinSqlFragments(conditions, ' AND ')}` : safeSql``
/**
* Calculates the chart bucketing level (minute/hour/day) given the date range.
@@ -274,10 +274,10 @@ const buildBaseWhere = (
if (excludeField !== 'log_type') {
const logTypeFilter = grouped.log_type
if (logTypeFilter) {
- const pred = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator)
- if (pred) parts.push(pred)
+ const condition = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator)
+ if (condition) parts.push(condition)
} else {
- parts.push(logTypeWherePredicate([...DEFAULT_LOG_TYPES]))
+ parts.push(logTypeWhereCondition([...DEFAULT_LOG_TYPES]))
}
}
@@ -285,10 +285,10 @@ const buildBaseWhere = (
if (key === excludeField) continue
if (key === 'log_type') continue // handled above
try {
- const predicate = translateFilter(key, values, operator)
- if (predicate) parts.push(predicate)
+ const condition = translateFilter(key, values, operator)
+ if (condition) parts.push(condition)
} catch {
- // analyticsLiteral rejected an unsupported input — drop the predicate.
+ // analyticsLiteral rejected an unsupported input — drop the condition.
}
}
@@ -296,9 +296,9 @@ const buildBaseWhere = (
}
/**
- * Returns a WHERE predicate that excludes Postgres connection lifecycle messages.
+ * Returns a WHERE condition that excludes Postgres connection lifecycle messages.
* Applied only to the row query and chart query so sidebar facets remain unaffected
- * (the OTEL endpoint's UNION ALL count query fails silently when this predicate is
+ * (the OTEL endpoint's UNION ALL count query fails silently when this condition is
* included there, returning empty facet data).
*/
const connectionLogsFilter = (search: QuerySearchParamsType): SafeLogSqlFragment | null => {
@@ -314,13 +314,13 @@ const connectionLogsFilter = (search: QuerySearchParamsType): SafeLogSqlFragment
* Unified logs row query — flat SELECT, no subquery wrapper.
*/
export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
- const predicates = buildBaseWhere(search)
+ const conditions = buildBaseWhere(search)
const connFilter = connectionLogsFilter(search)
- if (connFilter) predicates.push(connFilter)
+ if (connFilter) conditions.push(connFilter)
return safeSql`
SELECT ${rowProjection()}
FROM logs
-${whereClause(predicates)}
+${whereClause(conditions)}
`
}
@@ -355,76 +355,84 @@ export const getFacetCountQuery = ({
? ATTR.path
: safeSql`log_attributes[${lit(facet)}]`
- const predicates: SafeLogSqlFragment[] = [
+ const conditions: SafeLogSqlFragment[] = [
...buildBaseWhere(search, facet),
safeSql`(${facetExpr}) IS NOT NULL AND (${facetExpr}) != ''`,
]
if (facetSearch) {
- predicates.push(safeSql`(${facetExpr}) LIKE ${lit('%' + facetSearch + '%')}`)
+ conditions.push(safeSql`(${facetExpr}) LIKE ${lit('%' + facetSearch + '%')}`)
}
return safeSql`
-SELECT ${lit(facet)} AS dimension, (${facetExpr}) AS value, count() AS count
+SELECT ${lit(facet)} AS facet, (${facetExpr}) AS value, count() AS count
FROM logs
-${whereClause(predicates)}
+${whereClause(conditions)}
GROUP BY value
LIMIT ${lit(MAX_FACETS_QUANTITY)}
`
}
/**
- * Bundled count query — UNION ALL of (dimension, value, count) rows so the
- * frontend can render facet counts and total in one round trip.
+ * Builds the facet-count query for the logs sidebar. Each row is
+ * (facet, value, count): for one filter category (e.g. level) and one of its
+ * values (e.g. warning), how many logs matched. The special `total` facet
+ * counts every matching log for the count badge.
*/
export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
- // When no predicates remain, fall back to `1` so we emit a valid
- // tautology rather than a bare `WHERE`.
- const baseFiltersFor = (excludeField?: string): SafeLogSqlFragment => {
- const predicates = buildBaseWhere(search, excludeField)
- return predicates.length > 0 ? joinSqlFragments(predicates, ' AND ') : safeSql`1`
+ const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter))
+
+ const whereFor = (excludeField?: string): SafeLogSqlFragment => {
+ const conditions = buildBaseWhere(search, excludeField)
+ return conditions.length > 0 ? joinSqlFragments(conditions, ' AND ') : safeSql`1`
}
- // The "total" badge should reflect the user's *current* filter set,
- // including any active log_type filter. Pass no excludeField so the
- // log_type predicate is included.
- const totalSql = safeSql`
-SELECT 'total' AS dimension, 'all' AS value, count() AS count
-FROM logs
-WHERE ${baseFiltersFor()}
-`
+ const VALUE_EXPR = {
+ total: safeSql`'all'`,
+ log_type: LOG_TYPE_EXPR,
+ level: LEVEL_EXPR,
+ method: ATTR.method,
+ status: STATUS_EXPR,
+ }
- const logTypeBranches = joinSqlFragments(
- Object.entries(LOG_TYPE_PREDICATE).map(
- ([logType, predicate]) =>
- safeSql`
-SELECT 'log_type' AS dimension, ${lit(logType)} AS value, countIf(${predicate}) AS count
-FROM logs
-WHERE ${baseFiltersFor('log_type')}
-`
- ),
- ' UNION ALL '
- )
-
- const levelBranches = joinSqlFragments(
- (['success', 'warning', 'error'] as const).map(
- (lvl) =>
- safeSql`
-SELECT 'level' AS dimension, ${lit(lvl)} AS value, countIf((${LEVEL_EXPR}) = ${lit(lvl)}) AS count
+ const scanBlock = (
+ facets: (keyof typeof VALUE_EXPR)[],
+ where: SafeLogSqlFragment
+ ): SafeLogSqlFragment => {
+ const facetArray = joinSqlFragments(
+ facets.map((facet) => lit(facet)),
+ ','
+ )
+ const branches = facets.map((facet) => safeSql`facet = ${lit(facet)}, ${VALUE_EXPR[facet]}`)
+ return safeSql`
+SELECT
+ arrayJoin([${facetArray}]) AS facet,
+ multiIf(${joinSqlFragments([...branches, safeSql`''`], ', ')}) AS value,
+ count() AS count
FROM logs
-WHERE ${baseFiltersFor('level')}
+WHERE ${where}
+GROUP BY facet, value
+HAVING value != ''
`
- ),
- ' UNION ALL '
- )
-
- const facetBranches = joinSqlFragments(
- (['method', 'status', 'pathname'] as const).map(
- (facet) => safeSql`(${getFacetCountQuery({ search, facet })})`
- ),
- ' UNION ALL '
- )
-
- return joinSqlFragments([totalSql, logTypeBranches, levelBranches, facetBranches], ' UNION ALL ')
+ }
+
+ // log_type always scans alone: excluding it also drops the default-types
+ // filter, so its counts differ from the base scan even when unfiltered.
+ const blocks = [scanBlock(['log_type'], whereFor('log_type'))]
+
+ // A filtered facet gets its own scan so it can exclude its own filter and
+ // still count its other values; the rest share the base scan.
+ const baseFacets: (keyof typeof VALUE_EXPR)[] = ['total']
+ for (const facet of ['level', 'method', 'status'] as const) {
+ if (grouped[facet]) blocks.push(scanBlock([facet], whereFor(facet)))
+ else baseFacets.push(facet)
+ }
+ blocks.push(scanBlock(baseFacets, whereFor()))
+
+ // pathname is high-cardinality, so it needs its own LIMIT (the endpoint
+ // rejects LIMIT BY inside the shared arrayJoin).
+ blocks.push(safeSql`(${getFacetCountQuery({ search, facet: 'pathname' })})`)
+
+ return joinSqlFragments(blocks, ' UNION ALL ')
}
/**
@@ -433,9 +441,9 @@ WHERE ${baseFiltersFor('level')}
export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {
const truncationLevel = calculateChartBucketing(search)
const truncFn = truncationFunction(truncationLevel)
- const predicates = buildBaseWhere(search)
+ const conditions = buildBaseWhere(search)
const connFilter = connectionLogsFilter(search)
- if (connFilter) predicates.push(connFilter)
+ if (connFilter) conditions.push(connFilter)
return safeSql`
SELECT
@@ -445,7 +453,7 @@ SELECT
countIf((${LEVEL_EXPR}) = 'error') AS error,
count() AS total_per_bucket
FROM logs
-${whereClause(predicates)}
+${whereClause(conditions)}
GROUP BY time_bucket
ORDER BY time_bucket ASC
`
diff --git a/apps/studio/data/logs/unified-logs-count-query.ts b/apps/studio/data/logs/unified-logs-count-query.ts
index 7fc3f0014ae..0a49dfbadd6 100644
--- a/apps/studio/data/logs/unified-logs-count-query.ts
+++ b/apps/studio/data/logs/unified-logs-count-query.ts
@@ -14,6 +14,10 @@ import { getLogsCountQuery as getLogsCountQueryBq } from '@/components/interface
import { FacetMetadataSchema } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.schema'
import { ResponseError, UseCustomQueryOptions } from '@/types'
+// Trimmed client-side because the OTEL endpoint rejects LIMIT BY, so the count
+// query can't cap rows per facet in SQL.
+const MAX_FACET_ROWS = 20
+
export async function getUnifiedLogsCount(
{ projectRef, search, useOtel = false }: UnifiedLogsVariables & { useOtel?: boolean },
signal?: AbortSignal
@@ -35,43 +39,41 @@ export async function getUnifiedLogsCount(
signal,
})
- // Process count results into facets structure
const facets: Record<string, FacetMetadataSchema> = {}
- const countsByDimension: Record<string, Map<string, number>> = {}
+ const countsByFacet: Record<string, Map<string, number>> = {}
let totalRowCount = 0
- // Group by dimension
if (data?.result) {
data.result.forEach((row: any) => {
- const dimension = row.dimension
+ const facet = row.facet
const value = row.value
const count = Number(row.count || 0)
- // Set total count if this is the total dimension
- if (dimension === 'total' && value === 'all') {
+ if (facet === 'total' && value === 'all') {
totalRowCount = count
}
- // Initialize dimension map if not exists
- if (!countsByDimension[dimension]) {
- countsByDimension[dimension] = new Map()
+ if (!countsByFacet[facet]) {
+ countsByFacet[facet] = new Map()
}
- // Add count to the dimension map
- countsByDimension[dimension].set(value, count)
+ countsByFacet[facet].set(value, count)
})
}
- // Convert dimension maps to facets structure
- Object.entries(countsByDimension).forEach(([dimension, countsMap]) => {
- // Skip the 'total' dimension as it's not a facet
- if (dimension === 'total') return
+ Object.entries(countsByFacet).forEach(([facet, countsMap]) => {
+ if (facet === 'total') return
+
+ const rows = Array.from(countsMap.entries())
+ .map(([value, count]) => ({ value, total: count }))
+ .sort((a, b) => b.total - a.total)
+ .slice(0, MAX_FACET_ROWS)
- const dimensionTotal = Array.from(countsMap.values()).reduce((sum, count) => sum + count, 0)
+ const facetTotal = rows.reduce((sum, row) => sum + row.total, 0)
- facets[dimension] = {
- total: dimensionTotal,
- rows: Array.from(countsMap.entries()).map(([value, count]) => ({ value, total: count })),
+ facets[facet] = {
+ total: facetTotal,
+ rows,
}
})