Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Query optimization (#950)

Goldflag8aaf763b602d
diff --git a/client/src/app/[site]/main/page.tsx b/client/src/app/[site]/main/page.tsx
index 1ad53f111..a6d2909ac 100644
--- a/client/src/app/[site]/main/page.tsx
+++ b/client/src/app/[site]/main/page.tsx
@@ -1,5 +1,7 @@
"use client";
+import { ReactNode } from "react";
import { useGetLiveUserCount } from "../../../api/analytics/hooks/useGetLiveUserCount";
+import { useInView } from "../../../hooks/useInView";
import { useSetPageTitle } from "../../../hooks/useSetPageTitle";
import { IS_CLOUD } from "../../../lib/const";
import { useStore } from "../../../lib/store";
@@ -14,6 +16,15 @@ import { Referrers } from "./components/sections/Referrers";
import { SearchConsole } from "./components/sections/SearchConsole";
import { Weekdays } from "./components/sections/Weekdays";
+function LazySection({ children, height = "405px" }: { children: ReactNode; height?: string }) {
+ const { ref, isInView } = useInView({ persistVisibility: true, rootMargin: "100px 0px" });
+ return (
+ <div ref={ref} style={{ minHeight: isInView ? undefined : height }}>
+ {isInView ? children : null}
+ </div>
+ );
+}
+
export default function MainPage() {
const { site } = useStore();
@@ -34,14 +45,14 @@ function MainPageContent() {
<SubHeader />
<MainSection />
<div className="grid grid-cols-1 lg:grid-cols-2 gap-3 mt-3">
- <Referrers />
- <Pages />
- <Devices />
- <Countries />
- <Events />
- <Weekdays />
- {IS_CLOUD && <Network />}
- {IS_CLOUD && <SearchConsole />}
+ <LazySection><Referrers /></LazySection>
+ <LazySection><Pages /></LazySection>
+ <LazySection><Devices /></LazySection>
+ <LazySection><Countries /></LazySection>
+ <LazySection height="483px"><Events /></LazySection>
+ <LazySection><Weekdays /></LazySection>
+ {IS_CLOUD && <LazySection><Network /></LazySection>}
+ {IS_CLOUD && <LazySection><SearchConsole /></LazySection>}
</div>
</div>
);
diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml
index a540d657c..4e9dea140 100644
--- a/docker-compose.cloud.yml
+++ b/docker-compose.cloud.yml
@@ -201,7 +201,11 @@ configs:
<console>true</console>
</logger>
<query_thread_log remove="remove"/>
- <query_log remove="remove"/>
+ <query_log>
+ <database>system</database>
+ <table>query_log</table>
+ <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+ </query_log>
<text_log remove="remove"/>
<trace_log remove="remove"/>
<metric_log remove="remove"/>
@@ -217,7 +221,7 @@ configs:
<clickhouse>
<profiles>
<default>
- <log_queries>0</log_queries>
+ <log_queries>1</log_queries>
<log_query_threads>0</log_query_threads>
<log_processors_profiles>0</log_processors_profiles>
<max_memory_usage>8000000000</max_memory_usage>
diff --git a/server/src/api/analytics/events/getOutboundLinks.ts b/server/src/api/analytics/events/getOutboundLinks.ts
index 9e196baee..2bc4398c9 100644
--- a/server/src/api/analytics/events/getOutboundLinks.ts
+++ b/server/src/api/analytics/events/getOutboundLinks.ts
@@ -26,15 +26,18 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksReque
const query = `
SELECT
- toString(props) as properties,
- timestamp
+ JSONExtractString(toString(props), 'url') AS url,
+ COUNT(*) AS count,
+ toString(MAX(timestamp)) AS lastClicked
FROM events
WHERE
site_id = {siteId:Int32}
AND type = 'outbound'
${timeStatement}
${filterStatement}
- ORDER BY timestamp DESC
+ GROUP BY url
+ ORDER BY count DESC
+ LIMIT 1000
`;
try {
@@ -46,41 +49,7 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksReque
},
});
- interface RawOutboundEvent {
- properties: string;
- timestamp: string;
- }
-
- const rawData = await processResults<RawOutboundEvent>(result);
-
- const urlCounts = new Map<string, { count: number; lastClicked: string }>();
-
- rawData.forEach(event => {
- try {
- const props = JSON.parse(event.properties);
- const url = props.url;
- if (url) {
- const existing = urlCounts.get(url);
- if (existing) {
- existing.count++;
- // Keep the most recent timestamp
- if (event.timestamp > existing.lastClicked) {
- existing.lastClicked = event.timestamp;
- }
- } else {
- urlCounts.set(url, { count: 1, lastClicked: event.timestamp });
- }
- }
- } catch (e) {
- // Skip events with invalid JSON
- }
- });
-
- // Convert to response format
- const data: GetOutboundLinksResponse = Array.from(urlCounts.entries())
- .map(([url, { count, lastClicked }]) => ({ url, count, lastClicked }))
- .sort((a, b) => b.count - a.count); // Sort by count descending
-
+ const data = await processResults<GetOutboundLinksResponse[number]>(result);
return res.send({ data });
} catch (error) {
console.error("Generated Query:", query);
diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.ts
index 4dada6b52..d0a013fc0 100644
--- a/server/src/api/analytics/getMetric.ts
+++ b/server/src/api/analytics/getMetric.ts
@@ -352,6 +352,8 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole
// Default case for other parameters
const sqlParam = getSqlParam(parameter);
+ const hasFilter = filterStatement.trim().length > 0;
+
if (isCountQuery) {
return `
SELECT COUNT(DISTINCT ${sqlParam}) as totalCount
@@ -365,6 +367,41 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole
`;
}
+ if (!hasFilter) {
+ // Single scan: compute pageviews_in_session inline via countIf per session.
+ // Safe because with no filter, the scan covers all events in the session anyway.
+ return `
+ WITH SessionData AS (
+ SELECT
+ argMin(${sqlParam}, timestamp) as value,
+ session_id,
+ countIf(type = 'pageview') as pageviews_in_session
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ AND ${sqlParam} IS NOT NULL
+ AND ${sqlParam} <> ''
+ ${timeStatement}
+ GROUP BY session_id
+ )
+ SELECT
+ value,
+ COUNT(DISTINCT session_id) as count,
+ round((COUNT(DISTINCT session_id) / sum(COUNT(DISTINCT session_id)) OVER ()) * 100, 2) as percentage,
+ COUNT() as pageviews,
+ round((COUNT() / sum(COUNT()) OVER ()) * 100, 2) as pageviews_percentage,
+ round((countIf(DISTINCT session_id, pageviews_in_session = 1) / nullIf(COUNT(DISTINCT session_id), 0)) * 100, 2) as bounce_rate
+ FROM SessionData
+ GROUP BY value
+ ORDER BY count desc
+ ${limitStatement}
+ ${offsetStatement};
+ `;
+ }
+
+ // With filters: two-scan approach preserves correct bounce rate.
+ // SessionPageCounts counts total pageviews per session (unfiltered) so bounce rate
+ // reflects the true session behaviour, not just filtered events.
return `
WITH SessionPageCounts AS (
SELECT
diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.ts
index 5a84ff83e..0ad3c67fa 100644
--- a/server/src/api/analytics/getOverview.ts
+++ b/server/src/api/analytics/getOverview.ts
@@ -16,73 +16,72 @@ type GetOverviewResponse = {
const getQuery = (params: FilterParams, siteId: number) => {
const timeStatement = getTimeStatement(params);
const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);
+ const hasFilter = filterStatement.trim().length > 0;
+ if (!hasFilter) {
+ // Single scan: no filter means AllSessionPageviews === FilteredSessions
+ return `
+ WITH SessionMetrics AS (
+ SELECT
+ session_id,
+ anyLast(user_id) AS user_id,
+ MIN(timestamp) AS start_time,
+ MAX(timestamp) AS end_time,
+ countIf(type = 'pageview') AS total_pageviews_in_session
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ ${timeStatement}
+ GROUP BY session_id
+ )
+ SELECT
+ COUNT() AS sessions,
+ AVG(total_pageviews_in_session) AS pages_per_session,
+ sumIf(1, total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,
+ AVG(end_time - start_time) AS session_duration,
+ SUM(total_pageviews_in_session) AS pageviews,
+ COUNT(DISTINCT user_id) AS users
+ FROM SessionMetrics`;
+ }
+
+ // Two scans (down from three):
+ // 1. AllSessionPageviews — unfiltered, for accurate bounce rate / pages_per_session
+ // 2. FilteredSessionsWithStats — filtered, provides session timing + pageviews + users
return `
WITH
- -- First, calculate total pageviews per session (no filters except time)
AllSessionPageviews AS (
SELECT
session_id,
- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS total_pageviews_in_session
+ countIf(type = 'pageview') AS total_pageviews_in_session
FROM events
WHERE
site_id = {siteId:Int32}
${timeStatement}
GROUP BY session_id
),
- -- Then get session data with filters applied
- FilteredSessions AS (
+ FilteredSessionsWithStats AS (
SELECT
session_id,
+ anyLast(user_id) AS user_id,
MIN(timestamp) AS start_time,
- MAX(timestamp) AS end_time
+ MAX(timestamp) AS end_time,
+ countIf(type = 'pageview') AS filtered_pageviews
FROM events
WHERE
site_id = {siteId:Int32}
${filterStatement}
${timeStatement}
GROUP BY session_id
- ),
- -- Join to get sessions with their total pageviews
- SessionsWithPageviews AS (
- SELECT
- fs.session_id,
- fs.start_time,
- fs.end_time,
- asp.total_pageviews_in_session
- FROM FilteredSessions fs
- LEFT JOIN AllSessionPageviews asp ON fs.session_id = asp.session_id
)
SELECT
- session_stats.sessions,
- session_stats.pages_per_session,
- session_stats.bounce_rate * 100 AS bounce_rate,
- session_stats.session_duration,
- page_stats.pageviews,
- page_stats.users
- FROM
- (
- -- Session-level metrics
- SELECT
- COUNT() AS sessions,
- AVG(total_pageviews_in_session) AS pages_per_session,
- sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,
- AVG(end_time - start_time) AS session_duration
- FROM SessionsWithPageviews
- ) AS session_stats
- CROSS JOIN
- (
- -- Page-level and user-level metrics
- SELECT
- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pageviews,
- COUNT(DISTINCT user_id) AS users
- FROM events
- WHERE
- site_id = {siteId:Int32}
- ${filterStatement}
- ${timeStatement}
- -- AND type = 'pageview'
- ) AS page_stats`;
+ COUNT() AS sessions,
+ AVG(asp.total_pageviews_in_session) AS pages_per_session,
+ sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,
+ AVG(f.end_time - f.start_time) AS session_duration,
+ SUM(f.filtered_pageviews) AS pageviews,
+ COUNT(DISTINCT f.user_id) AS users
+ FROM FilteredSessionsWithStats f
+ LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;
};
export interface OverviewRequest {
diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.ts
index c5ae63de8..148f5eff7 100644
--- a/server/src/api/analytics/getOverviewBucketed.ts
+++ b/server/src/api/analytics/getOverviewBucketed.ts
@@ -42,7 +42,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
const startIso = startTimestamp.toISOString().slice(0, 19).replace("T", " ");
const endIso = endTimestamp.toISOString().slice(0, 19).replace("T", " ");
- return ` WITH FILL
+ return ` WITH FILL
FROM ${TimeBucketToFn[validatedBucket]}(toDateTime(${SqlString.escape(startIso)}))
TO ${TimeBucketToFn[validatedBucket]}(toDateTime(${SqlString.escape(endIso)})) + INTERVAL 1 ${
validatedBucket === "minute"
@@ -70,6 +70,7 @@ const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number)
const { start_date, end_date, time_zone, bucket = "hour", filters, past_minutes_start, past_minutes_end } = params;
const timeStatement = getTimeStatement(params);
const filterStatement = getFilterStatement(filters, siteId, timeStatement);
+ const hasFilter = filterStatement.trim().length > 0;
const pastMinutesRange =
past_minutes_start !== undefined && past_minutes_end !== undefined
@@ -77,10 +78,66 @@ const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number)
: undefined;
const isAllTime = !start_date && !end_date && !pastMinutesRange;
+ const fillClause = isAllTime ? "" : getTimeStatementFill(params, bucket);
+ const tzEscaped = SqlString.escape(time_zone);
- const query = `
+ if (!hasFilter) {
+ // No filter: merge AllSessionPageviews + FilteredSessions into one session scan.
+ // A second scan handles pageviews/users bucketed by event timestamp for accuracy.
+ return `
+WITH SessionMetrics AS (
+ SELECT
+ session_id,
+ anyLast(user_id) AS user_id,
+ MIN(timestamp) AS start_time,
+ MAX(timestamp) AS end_time,
+ countIf(type = 'pageview') AS total_pageviews_in_session
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ ${timeStatement}
+ GROUP BY session_id
+),
+SessionBucketed AS (
+ SELECT
+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${tzEscaped}))) AS time,
+ COUNT() AS sessions,
+ AVG(total_pageviews_in_session) AS pages_per_session,
+ sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,
+ AVG(end_time - start_time) AS session_duration,
+ COUNT(DISTINCT user_id) AS users
+ FROM SessionMetrics
+ GROUP BY time
+ ORDER BY time ${fillClause}
+),
+EventBucketed AS (
+ SELECT
+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${tzEscaped}))) AS time,
+ countIf(type = 'pageview') AS pageviews
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ ${timeStatement}
+ GROUP BY time
+ ORDER BY time ${fillClause}
+)
+SELECT
+ s.time AS time,
+ s.sessions,
+ s.pages_per_session,
+ s.bounce_rate * 100 AS bounce_rate,
+ s.session_duration,
+ e.pageviews,
+ s.users
+FROM SessionBucketed s
+FULL JOIN EventBucketed e USING time
+ORDER BY time`;
+ }
+
+ // With filters: keep original 3-CTE structure (semantics for bounce rate require
+ // unfiltered pageview counts per session from AllSessionPageviews).
+ return `
WITH
--- First, calculate total pageviews per session (no parameter filters)
AllSessionPageviews AS (
SELECT
session_id,
@@ -91,7 +148,6 @@ AllSessionPageviews AS (
${getTimeStatement(params)}
GROUP BY session_id
),
--- Then get session data with filters applied
FilteredSessions AS (
SELECT
session_id,
@@ -104,7 +160,6 @@ FilteredSessions AS (
${getTimeStatement(params)}
GROUP BY session_id
),
--- Join to get sessions with their total pageviews
SessionsWithPageviews AS (
SELECT
fs.session_id,
@@ -125,18 +180,18 @@ SELECT
FROM
(
SELECT
- toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${SqlString.escape(time_zone)}))) AS time,
+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(start_time, ${tzEscaped}))) AS time,
COUNT() AS sessions,
AVG(total_pageviews_in_session) AS pages_per_session,
sumIf(1, total_pageviews_in_session = 1) / COUNT() AS bounce_rate,
AVG(end_time - start_time) AS session_duration
FROM SessionsWithPageviews
- GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, bucket)}
+ GROUP BY time ORDER BY time ${fillClause}
) AS session_stats
FULL JOIN
(
SELECT
- toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${SqlString.escape(time_zone)}))) AS time,
+ toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, ${tzEscaped}))) AS time,
countIf(type = 'pageview') AS pageviews,
COUNT(DISTINCT user_id) AS users
FROM events
@@ -144,12 +199,10 @@ FULL JOIN
site_id = {siteId:Int32}
${filterStatement}
${getTimeStatement(params)}
- GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, bucket)}
+ GROUP BY time ORDER BY time ${fillClause}
) AS page_stats
USING time
ORDER BY time`;
-
- return query;
};
type getOverviewBucketed = { time: string; pageviews: number }[];
diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.ts
index bdf06a2f5..2891f93ab 100644
--- a/server/src/api/analytics/sessions/getSessions.ts
+++ b/server/src/api/analytics/sessions/getSessions.ts
@@ -2,7 +2,7 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
-import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { getFilterStatement, getEventLevelFilterStatement } from "../utils/getFilterStatement.js";
export type GetSessionsResponse = {
session_id: string;
@@ -103,6 +103,11 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:
fieldMappings: SESSION_FIELD_MAPPINGS,
});
+ // Inner filter: only raw event-level columns (country, browser, device_type, etc.)
+ // Applied inside the CTE before GROUP BY to reduce rows aggregated.
+ // The outer filterStatement still enforces full correctness.
+ const innerFilterStatement = getEventLevelFilterStatement(filters, Number(site), timeStatement);
+
const query = `
WITH AggregatedSessions AS (
SELECT
@@ -151,6 +156,7 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:
${userId ? ` AND (events.user_id = {user_id:String} OR events.identified_user_id = {user_id:String})` : ""}
${sessionId ? ` AND events.session_id = {session_id:String}` : ""}
${timeStatement}
+ ${innerFilterStatement}
GROUP BY
session_id
ORDER BY session_end DESC
diff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.ts
index 398cc6815..55e964be8 100644
--- a/server/src/api/analytics/utils/getFilterStatement.ts
+++ b/server/src/api/analytics/utils/getFilterStatement.ts
@@ -11,8 +11,31 @@ export interface FilterStatementOptions {
// Field name mappings for CTEs that extract fields to different column names
// e.g., { "url_parameters['utm_source']": "utm_source" }
fieldMappings?: Record<string, string>;
+
+ // When set, only generate conditions for the listed parameters (all others are skipped)
+ includeOnly?: FilterParameter[];
}
+// Raw event-level columns that have consistent values across all events in a session.
+// Safe to push into a CTE WHERE clause before GROUP BY session_id.
+export const INNER_SAFE_PARAMS: FilterParameter[] = [
+ "country",
+ "region",
+ "city",
+ "language",
+ "device_type",
+ "browser",
+ "browser_version",
+ "operating_system",
+ "operating_system_version",
+ "hostname",
+ "referrer",
+ "channel",
+ "dimensions",
+ "lat",
+ "lon",
+];
+
const DEFAULT_SESSION_LEVEL_PARAMS: FilterParameter[] = ["event_name"];
const filterTypeToOperator = (type: FilterType) => {
@@ -94,6 +117,7 @@ export function getFilterStatement(
}
const sessionLevelParams = options?.sessionLevelParams ?? DEFAULT_SESSION_LEVEL_PARAMS;
+ const includeOnly = options?.includeOnly;
const siteIdFilter = siteId ? `site_id = ${siteId}` : "";
// Strip leading "AND " from timeStatement since we'll be constructing WHERE clauses
const timeFilter = timeStatement ? timeStatement.replace(/^AND\s+/i, "").trim() : "";
@@ -120,9 +144,17 @@ export function getFilterStatement(
)`;
};
+ const activeFilters = includeOnly
+ ? filtersArray.filter(f => includeOnly.includes(f.parameter))
+ : filtersArray;
+
+ if (activeFilters.length === 0) {
+ return "";
+ }
+
let result =
"AND " +
- filtersArray
+ activeFilters
.map(filter => {
const x = filter.type === "contains" || filter.type === "not_contains" ? "%" : "";
const isNumericParam = filter.parameter === "lat" || filter.parameter === "lon";
@@ -312,3 +344,16 @@ export function getFilterStatement(
return result;
}
+
+// Returns a WHERE fragment containing only raw event-level column filters.
+// Safe to push inside a CTE WHERE clause before GROUP BY session_id, because
+// these columns have consistent values across all events in a session.
+export function getEventLevelFilterStatement(
+ filters: string,
+ siteId?: number,
+ timeStatement?: string
+): string {
+ return getFilterStatement(filters, siteId, timeStatement, {
+ includeOnly: INNER_SAFE_PARAMS,
+ });
+}