The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
perf(studio): single-scan unified logs facet count query (#47088)
Jordi Enricb87110b695eb
diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.tsindex 5d9d9f8380f..ba73e93c0b4 100644--- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts+++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.test.ts@@ -134,29 +134,40 @@ describe('UnifiedLogs.queries (OTEL flat)', () => {})describe('getLogsCountQuery', () => {- it('emits one UNION ALL branch per log_type bucket and per level', () => {+ const whereOfBranchContaining = (sql: string, needle: string) => {+ const branch = sql.split(/\bUNION ALL\b/).find((b) => b.includes(needle)) ?? ''+ return branch.split(/\bWHERE\b/)[1]?.split(/\bGROUP BY\b/)[0] ?? ''+ }++ it('folds facets into single-pass arrayJoin scans with a total row', () => {const sql = getLogsCountQuery(baseSearch)- // Per-log-type counts+ expect(sql).toContain('arrayJoin([')+ expect(sql).toContain('multiIf(')+ expect(sql).toContain(`facet = 'total', 'all'`)for (const lt of ['edge', 'postgrest', 'storage', 'postgres', 'edge function', 'auth']) {expect(sql).toContain(`'${lt}'`)}- // Per-level countsfor (const lvl of ['success', 'warning', 'error']) {expect(sql).toContain(`'${lvl}'`)}- // Bundled via UNION ALL — multiple occurrences expected- expect(sql.match(/UNION ALL/g)?.length ?? 0).toBeGreaterThan(5)+ expect(sql).toContain(`'pathname'`)+ expect(sql).toContain('LIMIT 20')+ // log_type + base + pathname = 3 scans+ expect(sql.match(/FROM logs/g)?.length ?? 0).toBeLessThanOrEqual(4)+ })++ it('honours an active log_type filter in the total count scan', () => {+ const sql = getLogsCountQuery(withFilters('log_type:eq:edge'))+ // Assert on the WHERE only: value expressions mention other sources inline.+ const totalWhere = whereOfBranchContaining(sql, `'all'`)+ expect(totalWhere).toContain(`source = 'edge_logs'`)+ expect(totalWhere).not.toContain(`source = 'postgres_logs'`)})- it('honours an active log_type filter in the total count branch', () => {+ it('gives the log_type facet its own scan that excludes the log_type filter', () => {const sql = getLogsCountQuery(withFilters('log_type:eq:edge'))- // The first branch is the total — its WHERE must include the edge- // log_type predicate, otherwise the total badge would over-count- // when a log_type filter is active.- const totalBranch = sql.split(/\bUNION ALL\b/)[0]- expect(totalBranch).toContain(`'total'`)- expect(totalBranch).toContain(`source = 'edge_logs'`)- expect(totalBranch).not.toContain(`source = 'postgres_logs'`)+ const logTypeWhere = whereOfBranchContaining(sql, `'log_type'`)+ expect(logTypeWhere).not.toContain(`NOT LIKE '%/rest/%'`)})})diff --git a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.tsindex 39ab733e62e..30121d41b21 100644--- a/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts+++ b/apps/studio/components/interfaces/UnifiedLogs/UnifiedLogs.queries.ts@@ -45,7 +45,7 @@ const ATTR = {const HTTP_STATUS_EXPR: SafeLogSqlFragment = safeSql`if(source = 'auth_logs', log_attributes['status'], ${ATTR.status})`/**- * Predicate that matches rows belonging to a given log_type. Mirrors the+ * Condition that matches rows belonging to a given log_type. Mirrors the* shape of the original BigQuery unified-logs CTEs: edge gateway traffic* (`source = 'edge_logs'`) is split between `edge`, `postgrest` and `storage`* based on URL path. Other types map straight to a single source.@@ -54,7 +54,7 @@ const HTTP_STATUS_EXPR: SafeLogSqlFragment = safeSql`if(source = 'auth_logs', lo* logs from postgREST / storage-api and are intentionally not part of unified* logs; the UI surfaces gateway HTTP traffic for those buckets.*/-const LOG_TYPE_PREDICATE: Record<string, SafeLogSqlFragment> = {+const LOG_TYPE_CONDITION: Record<string, SafeLogSqlFragment> = {edge: safeSql`source = 'edge_logs' AND ${ATTR.path} NOT LIKE '%/rest/%' AND ${ATTR.path} NOT LIKE '%/storage/%'`,postgrest: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/rest/%'`,storage: safeSql`source = 'edge_logs' AND ${ATTR.path} LIKE '%/storage/%'`,@@ -106,18 +106,18 @@ const LEVEL_EXPR: SafeLogSqlFragment = safeSql`CASEELSE 'success'END`-const logTypeWherePredicate = (logTypes: string[]): SafeLogSqlFragment => {- const effective = logTypes.filter((t) => t in LOG_TYPE_PREDICATE)+const logTypeWhereCondition = (logTypes: string[]): SafeLogSqlFragment => {+ const effective = logTypes.filter((t) => t in LOG_TYPE_CONDITION)const types = effective.length ? effective : [...DEFAULT_LOG_TYPES]- const branches = types.map((t) => safeSql`(${LOG_TYPE_PREDICATE[t]})`)+ const branches = types.map((t) => safeSql`(${LOG_TYPE_CONDITION[t]})`)return safeSql`(${joinSqlFragments(branches, ' OR ')})`}/*** Translates one (column, values, operator) group from the parsed `filter` URL- * param into an underlying SQL predicate. The OTEL endpoint rejects queries+ * param into an underlying SQL condition. The OTEL endpoint rejects queries* that reference derived aliases like `log_type` or `level` in WHERE for some- * shapes, so we always emit raw-column predicates (source/severity_text/+ * shapes, so we always emit raw-column conditions (source/severity_text/* log_attributes[…]). For `<>`, IN becomes NOT IN, LIKE becomes NOT LIKE, and* multi-value lists are joined with AND so the row must not match *any* value.*/@@ -142,8 +142,8 @@ const translateFilter = (switch (key) {case 'log_type': {const branches = values.map((t) => {- const pred = LOG_TYPE_PREDICATE[t] ?? safeSql`source = ${lit(t)}`- return isNeq ? safeSql`NOT (${pred})` : safeSql`(${pred})`+ const condition = LOG_TYPE_CONDITION[t] ?? safeSql`source = ${lit(t)}`+ return isNeq ? safeSql`NOT (${condition})` : safeSql`(${condition})`})return safeSql`(${joinSqlFragments(branches, joinAndOr)})`}@@ -193,8 +193,8 @@ const translateFilter = (}}-const whereClause = (predicates: SafeLogSqlFragment[]): SafeLogSqlFragment =>- predicates.length > 0 ? safeSql`WHERE ${joinSqlFragments(predicates, ' AND ')}` : safeSql``+const whereClause = (conditions: SafeLogSqlFragment[]): SafeLogSqlFragment =>+ conditions.length > 0 ? safeSql`WHERE ${joinSqlFragments(conditions, ' AND ')}` : safeSql``/*** Calculates the chart bucketing level (minute/hour/day) given the date range.@@ -274,10 +274,10 @@ const buildBaseWhere = (if (excludeField !== 'log_type') {const logTypeFilter = grouped.log_typeif (logTypeFilter) {- const pred = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator)- if (pred) parts.push(pred)+ const condition = translateFilter('log_type', logTypeFilter.values, logTypeFilter.operator)+ if (condition) parts.push(condition)} else {- parts.push(logTypeWherePredicate([...DEFAULT_LOG_TYPES]))+ parts.push(logTypeWhereCondition([...DEFAULT_LOG_TYPES]))}}@@ -285,10 +285,10 @@ const buildBaseWhere = (if (key === excludeField) continueif (key === 'log_type') continue // handled abovetry {- const predicate = translateFilter(key, values, operator)- if (predicate) parts.push(predicate)+ const condition = translateFilter(key, values, operator)+ if (condition) parts.push(condition)} catch {- // analyticsLiteral rejected an unsupported input — drop the predicate.+ // analyticsLiteral rejected an unsupported input — drop the condition.}}@@ -296,9 +296,9 @@ const buildBaseWhere = (}/**- * Returns a WHERE predicate that excludes Postgres connection lifecycle messages.+ * Returns a WHERE condition that excludes Postgres connection lifecycle messages.* Applied only to the row query and chart query so sidebar facets remain unaffected- * (the OTEL endpoint's UNION ALL count query fails silently when this predicate is+ * (the OTEL endpoint's UNION ALL count query fails silently when this condition is* included there, returning empty facet data).*/const connectionLogsFilter = (search: QuerySearchParamsType): SafeLogSqlFragment | null => {@@ -314,13 +314,13 @@ const connectionLogsFilter = (search: QuerySearchParamsType): SafeLogSqlFragment* Unified logs row query — flat SELECT, no subquery wrapper.*/export const getUnifiedLogsQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {- const predicates = buildBaseWhere(search)+ const conditions = buildBaseWhere(search)const connFilter = connectionLogsFilter(search)- if (connFilter) predicates.push(connFilter)+ if (connFilter) conditions.push(connFilter)return safeSql`SELECT ${rowProjection()}FROM logs-${whereClause(predicates)}+${whereClause(conditions)}`}@@ -355,76 +355,84 @@ export const getFacetCountQuery = ({? ATTR.path: safeSql`log_attributes[${lit(facet)}]`- const predicates: SafeLogSqlFragment[] = [+ const conditions: SafeLogSqlFragment[] = [...buildBaseWhere(search, facet),safeSql`(${facetExpr}) IS NOT NULL AND (${facetExpr}) != ''`,]if (facetSearch) {- predicates.push(safeSql`(${facetExpr}) LIKE ${lit('%' + facetSearch + '%')}`)+ conditions.push(safeSql`(${facetExpr}) LIKE ${lit('%' + facetSearch + '%')}`)}return safeSql`-SELECT ${lit(facet)} AS dimension, (${facetExpr}) AS value, count() AS count+SELECT ${lit(facet)} AS facet, (${facetExpr}) AS value, count() AS countFROM logs-${whereClause(predicates)}+${whereClause(conditions)}GROUP BY valueLIMIT ${lit(MAX_FACETS_QUANTITY)}`}/**- * Bundled count query — UNION ALL of (dimension, value, count) rows so the- * frontend can render facet counts and total in one round trip.+ * Builds the facet-count query for the logs sidebar. Each row is+ * (facet, value, count): for one filter category (e.g. level) and one of its+ * values (e.g. warning), how many logs matched. The special `total` facet+ * counts every matching log for the count badge.*/export const getLogsCountQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {- // When no predicates remain, fall back to `1` so we emit a valid- // tautology rather than a bare `WHERE`.- const baseFiltersFor = (excludeField?: string): SafeLogSqlFragment => {- const predicates = buildBaseWhere(search, excludeField)- return predicates.length > 0 ? joinSqlFragments(predicates, ' AND ') : safeSql`1`+ const grouped = groupLogsFiltersByColumn(parseLogsFilterUrlParams(search.filter))++ const whereFor = (excludeField?: string): SafeLogSqlFragment => {+ const conditions = buildBaseWhere(search, excludeField)+ return conditions.length > 0 ? joinSqlFragments(conditions, ' AND ') : safeSql`1`}- // The "total" badge should reflect the user's *current* filter set,- // including any active log_type filter. Pass no excludeField so the- // log_type predicate is included.- const totalSql = safeSql`-SELECT 'total' AS dimension, 'all' AS value, count() AS count-FROM logs-WHERE ${baseFiltersFor()}-`+ const VALUE_EXPR = {+ total: safeSql`'all'`,+ log_type: LOG_TYPE_EXPR,+ level: LEVEL_EXPR,+ method: ATTR.method,+ status: STATUS_EXPR,+ }- const logTypeBranches = joinSqlFragments(- Object.entries(LOG_TYPE_PREDICATE).map(- ([logType, predicate]) =>- safeSql`-SELECT 'log_type' AS dimension, ${lit(logType)} AS value, countIf(${predicate}) AS count-FROM logs-WHERE ${baseFiltersFor('log_type')}-`- ),- ' UNION ALL '- )-- const levelBranches = joinSqlFragments(- (['success', 'warning', 'error'] as const).map(- (lvl) =>- safeSql`-SELECT 'level' AS dimension, ${lit(lvl)} AS value, countIf((${LEVEL_EXPR}) = ${lit(lvl)}) AS count+ const scanBlock = (+ facets: (keyof typeof VALUE_EXPR)[],+ where: SafeLogSqlFragment+ ): SafeLogSqlFragment => {+ const facetArray = joinSqlFragments(+ facets.map((facet) => lit(facet)),+ ','+ )+ const branches = facets.map((facet) => safeSql`facet = ${lit(facet)}, ${VALUE_EXPR[facet]}`)+ return safeSql`+SELECT+ arrayJoin([${facetArray}]) AS facet,+ multiIf(${joinSqlFragments([...branches, safeSql`''`], ', ')}) AS value,+ count() AS countFROM logs-WHERE ${baseFiltersFor('level')}+WHERE ${where}+GROUP BY facet, value+HAVING value != ''`- ),- ' UNION ALL '- )-- const facetBranches = joinSqlFragments(- (['method', 'status', 'pathname'] as const).map(- (facet) => safeSql`(${getFacetCountQuery({ search, facet })})`- ),- ' UNION ALL '- )-- return joinSqlFragments([totalSql, logTypeBranches, levelBranches, facetBranches], ' UNION ALL ')+ }++ // log_type always scans alone: excluding it also drops the default-types+ // filter, so its counts differ from the base scan even when unfiltered.+ const blocks = [scanBlock(['log_type'], whereFor('log_type'))]++ // A filtered facet gets its own scan so it can exclude its own filter and+ // still count its other values; the rest share the base scan.+ const baseFacets: (keyof typeof VALUE_EXPR)[] = ['total']+ for (const facet of ['level', 'method', 'status'] as const) {+ if (grouped[facet]) blocks.push(scanBlock([facet], whereFor(facet)))+ else baseFacets.push(facet)+ }+ blocks.push(scanBlock(baseFacets, whereFor()))++ // pathname is high-cardinality, so it needs its own LIMIT (the endpoint+ // rejects LIMIT BY inside the shared arrayJoin).+ blocks.push(safeSql`(${getFacetCountQuery({ search, facet: 'pathname' })})`)++ return joinSqlFragments(blocks, ' UNION ALL ')}/**@@ -433,9 +441,9 @@ WHERE ${baseFiltersFor('level')}export const getLogsChartQuery = (search: QuerySearchParamsType): SafeLogSqlFragment => {const truncationLevel = calculateChartBucketing(search)const truncFn = truncationFunction(truncationLevel)- const predicates = buildBaseWhere(search)+ const conditions = buildBaseWhere(search)const connFilter = connectionLogsFilter(search)- if (connFilter) predicates.push(connFilter)+ if (connFilter) conditions.push(connFilter)return safeSql`SELECT@@ -445,7 +453,7 @@ SELECTcountIf((${LEVEL_EXPR}) = 'error') AS error,count() AS total_per_bucketFROM logs-${whereClause(predicates)}+${whereClause(conditions)}GROUP BY time_bucketORDER BY time_bucket ASC`diff --git a/apps/studio/data/logs/unified-logs-count-query.ts b/apps/studio/data/logs/unified-logs-count-query.tsindex 7fc3f0014ae..0a49dfbadd6 100644--- a/apps/studio/data/logs/unified-logs-count-query.ts+++ b/apps/studio/data/logs/unified-logs-count-query.ts@@ -14,6 +14,10 @@ import { getLogsCountQuery as getLogsCountQueryBq } from '@/components/interfaceimport { FacetMetadataSchema } from '@/components/interfaces/UnifiedLogs/UnifiedLogs.schema'import { ResponseError, UseCustomQueryOptions } from '@/types'+// Trimmed client-side because the OTEL endpoint rejects LIMIT BY, so the count+// query can't cap rows per facet in SQL.+const MAX_FACET_ROWS = 20+export async function getUnifiedLogsCount({ projectRef, search, useOtel = false }: UnifiedLogsVariables & { useOtel?: boolean },signal?: AbortSignal@@ -35,43 +39,41 @@ export async function getUnifiedLogsCount(signal,})- // Process count results into facets structureconst facets: Record<string, FacetMetadataSchema> = {}- const countsByDimension: Record<string, Map<string, number>> = {}+ const countsByFacet: Record<string, Map<string, number>> = {}let totalRowCount = 0- // Group by dimensionif (data?.result) {data.result.forEach((row: any) => {- const dimension = row.dimension+ const facet = row.facetconst value = row.valueconst count = Number(row.count || 0)- // Set total count if this is the total dimension- if (dimension === 'total' && value === 'all') {+ if (facet === 'total' && value === 'all') {totalRowCount = count}- // Initialize dimension map if not exists- if (!countsByDimension[dimension]) {- countsByDimension[dimension] = new Map()+ if (!countsByFacet[facet]) {+ countsByFacet[facet] = new Map()}- // Add count to the dimension map- countsByDimension[dimension].set(value, count)+ countsByFacet[facet].set(value, count)})}- // Convert dimension maps to facets structure- Object.entries(countsByDimension).forEach(([dimension, countsMap]) => {- // Skip the 'total' dimension as it's not a facet- if (dimension === 'total') return+ Object.entries(countsByFacet).forEach(([facet, countsMap]) => {+ if (facet === 'total') return++ const rows = Array.from(countsMap.entries())+ .map(([value, count]) => ({ value, total: count }))+ .sort((a, b) => b.total - a.total)+ .slice(0, MAX_FACET_ROWS)- const dimensionTotal = Array.from(countsMap.values()).reduce((sum, count) => sum + count, 0)+ const facetTotal = rows.reduce((sum, row) => sum + row.total, 0)- facets[dimension] = {- total: dimensionTotal,- rows: Array.from(countsMap.entries()).map(([value, count]) => ({ value, total: count })),+ facets[facet] = {+ total: facetTotal,+ rows,}})