🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Query optimization (#950)
Goldflag8aaf763b602d
diff --git a/client/src/app/[site]/main/page.tsx b/client/src/app/[site]/main/page.tsxindex 1ad53f111..a6d2909ac 100644--- a/client/src/app/[site]/main/page.tsx+++ b/client/src/app/[site]/main/page.tsx@@ -1,5 +1,7 @@"use client";+import { ReactNode } from "react";import { useGetLiveUserCount } from "../../../api/analytics/hooks/useGetLiveUserCount";+import { useInView } from "../../../hooks/useInView";import { useSetPageTitle } from "../../../hooks/useSetPageTitle";import { IS_CLOUD } from "../../../lib/const";import { useStore } from "../../../lib/store";@@ -14,6 +16,15 @@ import { Referrers } from "./components/sections/Referrers";import { SearchConsole } from "./components/sections/SearchConsole";import { Weekdays } from "./components/sections/Weekdays";+function LazySection({ children, height = "405px" }: { children: ReactNode; height?: string }) {+ const { ref, isInView } = useInView({ persistVisibility: true, rootMargin: "100px 0px" });+ return (+ <div ref={ref} style={{ minHeight: isInView ? undefined : height }}>+ {isInView ? children : null}+ </div>+ );+}+export default function MainPage() {const { site } = useStore();@@ -34,14 +45,14 @@ function MainPageContent() {<SubHeader /><MainSection /><div className="grid grid-cols-1 lg:grid-cols-2 gap-3 mt-3">- <Referrers />- <Pages />- <Devices />- <Countries />- <Events />- <Weekdays />- {IS_CLOUD && <Network />}- {IS_CLOUD && <SearchConsole />}+ <LazySection><Referrers /></LazySection>+ <LazySection><Pages /></LazySection>+ <LazySection><Devices /></LazySection>+ <LazySection><Countries /></LazySection>+ <LazySection height="483px"><Events /></LazySection>+ <LazySection><Weekdays /></LazySection>+ {IS_CLOUD && <LazySection><Network /></LazySection>}+ {IS_CLOUD && <LazySection><SearchConsole /></LazySection>}</div></div>);diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.ymlindex a540d657c..4e9dea140 100644--- a/docker-compose.cloud.yml+++ b/docker-compose.cloud.yml@@ -201,7 +201,11 @@ configs:<console>true</console></logger><query_thread_log remove="remove"/>- <query_log remove="remove"/>+ <query_log>+ <database>system</database>+ <table>query_log</table>+ <flush_interval_milliseconds>7500</flush_interval_milliseconds>+ </query_log><text_log remove="remove"/><trace_log remove="remove"/><metric_log remove="remove"/>@@ -217,7 +221,7 @@ configs:<clickhouse><profiles><default>- <log_queries>0</log_queries>+ <log_queries>1</log_queries><log_query_threads>0</log_query_threads><log_processors_profiles>0</log_processors_profiles><max_memory_usage>8000000000</max_memory_usage>diff --git a/server/src/api/analytics/events/getOutboundLinks.ts b/server/src/api/analytics/events/getOutboundLinks.tsindex 9e196baee..2bc4398c9 100644--- a/server/src/api/analytics/events/getOutboundLinks.ts+++ b/server/src/api/analytics/events/getOutboundLinks.ts@@ -26,15 +26,18 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksRequeconst query = `SELECT- toString(props) as properties,- timestamp+ JSONExtractString(toString(props), 'url') AS url,+ COUNT(*) AS count,+ toString(MAX(timestamp)) AS lastClickedFROM eventsWHEREsite_id = {siteId:Int32}AND type = 'outbound'${timeStatement}${filterStatement}- ORDER BY timestamp DESC+ GROUP BY url+ ORDER BY count DESC+ LIMIT 1000`;try {@@ -46,41 +49,7 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksReque},});- interface RawOutboundEvent {- properties: string;- timestamp: string;- }-- const rawData = await processResults<RawOutboundEvent>(result);-- const urlCounts = new Map<string, { count: number; lastClicked: string }>();-- rawData.forEach(event => {- try {- const props = JSON.parse(event.properties);- const url = props.url;- if (url) {- const existing = urlCounts.get(url);- if (existing) {- existing.count++;- // Keep the most recent timestamp- if (event.timestamp > existing.lastClicked) {- existing.lastClicked = event.timestamp;- }- } else {- urlCounts.set(url, { count: 1, lastClicked: event.timestamp });- }- }- } catch (e) {- // Skip events with invalid JSON- }- });-- // Convert to response format- const data: GetOutboundLinksResponse = Array.from(urlCounts.entries())- .map(([url, { count, lastClicked }]) => ({ url, count, lastClicked }))- .sort((a, b) => b.count - a.count); // Sort by count descending-+ const data = await processResults<GetOutboundLinksResponse[number]>(result);return res.send({ data });} catch (error) {console.error("Generated Query:", query);diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.tsindex 4dada6b52..d0a013fc0 100644--- a/server/src/api/analytics/getMetric.ts+++ b/server/src/api/analytics/getMetric.ts@@ -352,6 +352,8 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole// Default case for other parametersconst sqlParam = getSqlParam(parameter);+ const hasFilter = filterStatement.trim().length > 0;+if (isCountQuery) {return `SELECT COUNT(DISTINCT ${sqlParam}) as totalCount@@ -365,6 +367,41 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole`;}+ if (!hasFilter) {+ // Single scan: compute pageviews_in_session inline via countIf per session.+ // Safe because with no filter, the scan covers all events in the session anyway.+ return `+ WITH SessionData AS (+ SELECT+ argMin(${sqlParam}, timestamp) as value,+ session_id,+ countIf(type = 'pageview') as pageviews_in_session+ FROM events+ WHERE+ site_id = {siteId:Int32}+ AND ${sqlParam} IS NOT NULL+ AND ${sqlParam} <> ''+ ${timeStatement}+ GROUP BY session_id+ )+ SELECT+ value,+ COUNT(DISTINCT session_id) as count,+ round((COUNT(DISTINCT session_id) / sum(COUNT(DISTINCT session_id)) OVER ()) * 100, 2) as percentage,+ COUNT() as pageviews,+ round((COUNT() / sum(COUNT()) OVER ()) * 100, 2) as pageviews_percentage,+ round((countIf(DISTINCT session_id, pageviews_in_session = 1) / nullIf(COUNT(DISTINCT session_id), 0)) * 100, 2) as bounce_rate+ FROM SessionData+ GROUP BY value+ ORDER BY count desc+ ${limitStatement}+ ${offsetStatement};+ `;+ }++ // With filters: two-scan approach preserves correct bounce rate.+ // SessionPageCounts counts total pageviews per session (unfiltered) so bounce rate+ // reflects the true session behaviour, not just filtered events.return `WITH SessionPageCounts AS (SELECTdiff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.tsindex 5a84ff83e..0ad3c67fa 100644--- a/server/src/api/analytics/getOverview.ts+++ b/server/src/api/analytics/getOverview.ts@@ -16,73 +16,72 @@ type GetOverviewResponse = {const getQuery = (params: FilterParams, siteId: number) => {const timeStatement = getTimeStatement(params);const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);+ const hasFilter = filterStatement.trim().length > 0;+ if (!hasFilter) {+ // Single scan: no filter means AllSessionPageviews === FilteredSessions+ return `+ WITH SessionMetrics AS (+ SELECT+ session_id,+ anyLast(user_id) AS user_id,+ MIN(timestamp) AS start_time,+ MAX(timestamp) AS end_time,+ countIf(type = 'pageview') AS total_pageviews_in_session+ FROM events+ WHERE+ site_id = {siteId:Int32}+ ${timeStatement}+ GROUP BY session_id+ )+ SELECT+ COUNT() AS sessions,+ AVG(total_pageviews_in_session) AS pages_per_session,+ sumIf(1, total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,+ AVG(end_time - start_time) AS session_duration,+ SUM(total_pageviews_in_session) AS pageviews,+ COUNT(DISTINCT user_id) AS users+ FROM SessionMetrics`;+ }++ // Two scans (down from three):+ // 1. AllSessionPageviews — unfiltered, for accurate bounce rate / pages_per_session+ // 2. FilteredSessionsWithStats — filtered, provides session timing + pageviews + usersreturn `WITH- -- First, calculate total pageviews per session (no filters except time)AllSessionPageviews AS (SELECTsession_id,- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS total_pageviews_in_session+ countIf(type = 'pageview') AS total_pageviews_in_sessionFROM eventsWHEREsite_id = {siteId:Int32}${timeStatement}GROUP BY session_id),- -- Then get session data with filters applied- FilteredSessions AS (+ FilteredSessionsWithStats AS (SELECTsession_id,+ anyLast(user_id) AS user_id,MIN(timestamp) AS start_time,- MAX(timestamp) AS end_time+ MAX(timestamp) AS end_time,+ countIf(type = 'pageview') AS filtered_pageviewsFROM eventsWHEREsite_id = {siteId:Int32}${filterStatement}${timeStatement}GROUP BY session_id- ),- -- Join to get sessions with their total pageviews- SessionsWithPageviews AS (- SELECT- fs.session_id,- fs.start_time,- fs.end_time,- asp.total_pageviews_in_session- FROM FilteredSessions fs- LEFT JOIN AllSessionPageviews asp ON fs.session_id = asp.session_id)SELECT- session_stats.sessions,- session_stats.pages_per_session,- session_stats.bounce_rate * 100 AS bounce_rate,- session_stats.session_duration,- page_stats.pageviews,- page_stats.users- FROM- (- -- Session-level metrics- SELECT- COUNT() AS sessions,- AVG(total_pageviews_in_session) AS pages_per_session,- sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,- AVG(end_time - start_time) AS session_duration- FROM SessionsWithPageviews- ) AS session_stats- CROSS JOIN- (- -- Page-level and user-level metrics- SELECT- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pageviews,- COUNT(DISTINCT user_id) AS users- FROM events- WHERE- site_id = {siteId:Int32}- ${filterStatement}- ${timeStatement}- -- AND type = 'pageview'- ) AS page_stats`;+ COUNT() AS sessions,+ AVG(asp.total_pageviews_in_session) AS pages_per_session,+ sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,+ AVG(f.end_time - f.start_time) AS session_duration,+ SUM(f.filtered_pageviews) AS pageviews,+ COUNT(DISTINCT f.user_id) AS users+ FROM FilteredSessionsWithStats f+ LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;};export interface OverviewRequest {diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.tsindex c5ae63de8..148f5eff7 100644--- a/server/src/api/analytics/getOverviewBucketed.ts+++ b/server/src/api/analytics/getOverviewBucketed.ts@@ -42,7 +42,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {const startIso = startTimestamp.toISOString().slice(0, 19).replace("T", " ");const endIso = endTimestamp.toISOString().slice(0, 19).replace("T", " ");- return ` WITH FILL+ return ` WITH FILLFROM ${TimeBucketToFn[validatedBucket]}(toDateTime(${SqlString.escape(startIso)}))TO ${TimeBucketToFn[validatedBucket]}(toDateTime(${SqlString.escape(endIso)})) + INTERVAL 1 ${validatedBucket === "minute"@@ -70,6 +70,7 @@ const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number)const { start_date, end_date, time_zone, bucket = "hour", filters, past_minutes_start, past_minutes_end } = params;const timeStatement = getTimeStatement(params);const filterStatement = getFilterStatement(filters, siteId, timeStatement);+ const hasFilter = filterStatement.trim().length > 0;const pastMinutesRange =past_minutes_start !== undefined && past_minutes_end !== undefined@@ -77,10 +78,66 @@ const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number): undefined;const isAllTime = !start_date && !end_date && !pastMinutesRange;+ const fillClause = isAllTime ? "" : getTimeStatementFill(params, bucket);+ const tzEscaped = SqlString.escape(time_zone);- const query = `+ if (!hasFilter) {+ // No filter: merge AllSessionPageviews + FilteredSessions into one session scan.+ // A second scan handles pageviews/users bucketed by event timestamp for accuracy.+ return `+WITH SessionMetrics AS (+ SELECT+ session_id,+ anyLast(user_id) AS user_id,+ MIN(timestamp) AS start_time,+ MAX(timestamp) AS end_time,+ countIf(type = 'pageview') AS total_pageviews_in_session+ FROM events+ WHERE+ site_id = {siteId:Int32}+ ${timeStatement}+ GROUP BY session_id+),+SessionBucketed AS (+ SELECT+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${tzEscaped}))) AS time,+ COUNT() AS sessions,+ AVG(total_pageviews_in_session) AS pages_per_session,+ sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,+ AVG(end_time - start_time) AS session_duration,+ COUNT(DISTINCT user_id) AS users+ FROM SessionMetrics+ GROUP BY time+ ORDER BY time ${fillClause}+),+EventBucketed AS (+ SELECT+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${tzEscaped}))) AS time,+ countIf(type = 'pageview') AS pageviews+ FROM events+ WHERE+ site_id = {siteId:Int32}+ ${timeStatement}+ GROUP BY time+ ORDER BY time ${fillClause}+)+SELECT+ s.time AS time,+ s.sessions,+ s.pages_per_session,+ s.bounce_rate * 100 AS bounce_rate,+ s.session_duration,+ e.pageviews,+ s.users+FROM SessionBucketed s+FULL JOIN EventBucketed e USING time+ORDER BY time`;+ }++ // With filters: keep original 3-CTE structure (semantics for bounce rate require+ // unfiltered pageview counts per session from AllSessionPageviews).+ return `WITH--- First, calculate total pageviews per session (no parameter filters)AllSessionPageviews AS (SELECTsession_id,@@ -91,7 +148,6 @@ AllSessionPageviews AS (${getTimeStatement(params)}GROUP BY session_id),--- Then get session data with filters appliedFilteredSessions AS (SELECTsession_id,@@ -104,7 +160,6 @@ FilteredSessions AS (${getTimeStatement(params)}GROUP BY session_id),--- Join to get sessions with their total pageviewsSessionsWithPageviews AS (SELECTfs.session_id,@@ -125,18 +180,18 @@ SELECTFROM(SELECT- toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${SqlString.escape(time_zone)}))) AS time,+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${tzEscaped}))) AS time,COUNT() AS sessions,AVG(total_pageviews_in_session) AS pages_per_session,sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,AVG(end_time - start_time) AS session_durationFROM SessionsWithPageviews- GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, bucket)}+ GROUP BY time ORDER BY time ${fillClause}) AS session_statsFULL JOIN(SELECT- toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${SqlString.escape(time_zone)}))) AS time,+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${tzEscaped}))) AS time,countIf(type = 'pageview') AS pageviews,COUNT(DISTINCT user_id) AS usersFROM events@@ -144,12 +199,10 @@ FULL JOINsite_id = {siteId:Int32}${filterStatement}${getTimeStatement(params)}- GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, bucket)}+ GROUP BY time ORDER BY time ${fillClause}) AS page_statsUSING timeORDER BY time`;-- return query;};type getOverviewBucketed = { time: string; pageviews: number }[];diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.tsindex bdf06a2f5..2891f93ab 100644--- a/server/src/api/analytics/sessions/getSessions.ts+++ b/server/src/api/analytics/sessions/getSessions.ts@@ -2,7 +2,7 @@ import { FastifyReply, FastifyRequest } from "fastify";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";-import { getFilterStatement } from "../utils/getFilterStatement.js";+import { getFilterStatement, getEventLevelFilterStatement } from "../utils/getFilterStatement.js";export type GetSessionsResponse = {session_id: string;@@ -103,6 +103,11 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:fieldMappings: SESSION_FIELD_MAPPINGS,});+ // Inner filter: only raw event-level columns (country, browser, device_type, etc.)+ // Applied inside the CTE before GROUP BY to reduce rows aggregated.+ // The outer filterStatement still enforces full correctness.+ const innerFilterStatement = getEventLevelFilterStatement(filters, Number(site), timeStatement);+const query = `WITH AggregatedSessions AS (SELECT@@ -151,6 +156,7 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:${userId ? ` AND (events.user_id = {user_id:String} OR events.identified_user_id = {user_id:String})` : ""}${sessionId ? ` AND events.session_id = {session_id:String}` : ""}${timeStatement}+ ${innerFilterStatement}GROUP BYsession_idORDER BY session_end DESCdiff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.tsindex 398cc6815..55e964be8 100644--- a/server/src/api/analytics/utils/getFilterStatement.ts+++ b/server/src/api/analytics/utils/getFilterStatement.ts@@ -11,8 +11,31 @@ export interface FilterStatementOptions {// Field name mappings for CTEs that extract fields to different column names// e.g., { "url_parameters['utm_source']": "utm_source" }fieldMappings?: Record<string, string>;++ // When set, only generate conditions for the listed parameters (all others are skipped)+ includeOnly?: FilterParameter[];}+// Raw event-level columns that have consistent values across all events in a session.+// Safe to push into a CTE WHERE clause before GROUP BY session_id.+export const INNER_SAFE_PARAMS: FilterParameter[] = [+ "country",+ "region",+ "city",+ "language",+ "device_type",+ "browser",+ "browser_version",+ "operating_system",+ "operating_system_version",+ "hostname",+ "referrer",+ "channel",+ "dimensions",+ "lat",+ "lon",+];+const DEFAULT_SESSION_LEVEL_PARAMS: FilterParameter[] = ["event_name"];const filterTypeToOperator = (type: FilterType) => {@@ -94,6 +117,7 @@ export function getFilterStatement(}const sessionLevelParams = options?.sessionLevelParams ?? DEFAULT_SESSION_LEVEL_PARAMS;+ const includeOnly = options?.includeOnly;const siteIdFilter = siteId ? `site_id = ${siteId}` : "";// Strip leading "AND " from timeStatement since we'll be constructing WHERE clausesconst timeFilter = timeStatement ? timeStatement.replace(/^AND\s+/i, "").trim() : "";@@ -120,9 +144,17 @@ export function getFilterStatement()`;};+ const activeFilters = includeOnly+ ? filtersArray.filter(f => includeOnly.includes(f.parameter))+ : filtersArray;++ if (activeFilters.length === 0) {+ return "";+ }+let result ="AND " +- filtersArray+ activeFilters.map(filter => {const x = filter.type === "contains" || filter.type === "not_contains" ? "%" : "";const isNumericParam = filter.parameter === "lat" || filter.parameter === "lon";@@ -312,3 +344,16 @@ export function getFilterStatement(return result;}++// Returns a WHERE fragment containing only raw event-level column filters.+// Safe to push inside a CTE WHERE clause before GROUP BY session_id, because+// these columns have consistent values across all events in a session.+export function getEventLevelFilterStatement(+ filters: string,+ siteId?: number,+ timeStatement?: string+): string {+ return getFilterStatement(filters, siteId, timeStatement, {+ includeOnly: INNER_SAFE_PARAMS,+ });+}