Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

refactor(analytics): one definition of the Site metrics, three surfaces that call it (#1115)

Goldflag8fc9917507e0
diff --git a/server/src/api/analytics/getOverview.test.ts b/server/src/api/analytics/getOverview.test.ts
index 6da6a5a7c..5a38bc18b 100644
--- a/server/src/api/analytics/getOverview.test.ts
+++ b/server/src/api/analytics/getOverview.test.ts
@@ -8,7 +8,6 @@ vi.mock("../../db/postgres/postgres.js", () => ({
}));
import { FilterParams } from "@rybbit/shared";
-import { buildOverviewQuery } from "./getOverview.js";
import { buildOverviewBucketedQuery } from "./getOverviewBucketed.js";
const SITE_ID = 1;
@@ -23,32 +22,9 @@ const baseParams = (overrides: Partial<Record<string, unknown>> = {}) =>
...overrides,
}) as FilterParams & { bucket: "day" };
+// The unbucketed overview is built by the Site Metrics module, which the PDF
+// report and the weekly email share; its coverage lives in siteMetrics.test.ts.
describe("unique user counting", () => {
- describe("buildOverviewQuery", () => {
- it("counts users by identity, falling back to the device fingerprint", () => {
- const sql = buildOverviewQuery(baseParams(), SITE_ID);
-
- expect(sql).toContain("COUNT(DISTINCT f.effective_user_id) AS users");
- expect(sql).toContain("anyIf(identified_user_id, identified_user_id != '')");
- expect(sql).toContain("anyLast(user_id)");
- });
-
- it("resolves the identity once per session rather than per event", () => {
- const sql = buildOverviewQuery(baseParams(), SITE_ID);
-
- // The identity expression must sit inside the session-grouped CTE, so a
- // visitor whose first pageview predates identify() still counts once.
- const sessionCteBody = sql.split("FilteredSessionsWithStats AS (")[1].split("GROUP BY session_id")[0];
- expect(sessionCteBody).toContain("anyIf(identified_user_id");
- });
-
- it("never aliases the identity back onto user_id", () => {
- // Shadowing a column with an alias used inside its own expression is a
- // cyclic-alias error in ClickHouse.
- expect(buildOverviewQuery(baseParams(), SITE_ID)).not.toMatch(/anyLast\(user_id\)\) AS user_id/);
- });
- });
-
describe("buildOverviewBucketedQuery", () => {
it("counts users by identity, falling back to the device fingerprint", () => {
const sql = buildOverviewBucketedQuery(baseParams(), SITE_ID);
diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.ts
index 3bd1fdc81..8d0f804ab 100644
--- a/server/src/api/analytics/getOverview.ts
+++ b/server/src/api/analytics/getOverview.ts
@@ -1,61 +1,18 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { getFilterStatement } from "./utils/getFilterStatement.js";
-import { getTimeStatement } from "./utils/utils.js";
+import {
+ buildMetricsSpec,
+ buildOverviewQuery as buildOverviewMetricsQuery,
+ OverviewRow,
+} from "../../services/siteMetrics/siteMetrics.js";
import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
-import { EFFECTIVE_SESSION_USER_ID } from "./utils/effectiveUserId.js";
-type GetOverviewResponse = {
- sessions: number;
- pageviews: number;
- users: number;
- pages_per_session: number;
- bounce_rate: number;
- session_duration: number;
-};
-
-export const buildOverviewQuery = (params: FilterParams, siteId: number) => {
- const timeStatement = getTimeStatement(params);
- const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);
-
- return `
- WITH
- AllSessionPageviews AS (
- SELECT
- session_id,
- countIf(type = 'pageview') AS total_pageviews_in_session
- FROM events
- WHERE
- site_id = {siteId:Int32}
- ${timeStatement}
- GROUP BY session_id
- ),
- FilteredSessionsWithStats AS (
- SELECT
- session_id,
- -- Aliased away from \`user_id\` on purpose: an alias that shadows a column
- -- referenced inside its own expression is a cyclic-alias error in ClickHouse.
- ${EFFECTIVE_SESSION_USER_ID} AS effective_user_id,
- MIN(timestamp) AS start_time,
- MAX(timestamp) AS end_time,
- countIf(type = 'pageview') AS filtered_pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- ${filterStatement}
- ${timeStatement}
- GROUP BY session_id
- )
- SELECT
- COUNT() AS sessions,
- AVG(asp.total_pageviews_in_session) AS pages_per_session,
- sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,
- AVG(f.end_time - f.start_time) AS session_duration,
- SUM(f.filtered_pageviews) AS pageviews,
- COUNT(DISTINCT f.effective_user_id) AS users
- FROM FilteredSessionsWithStats f
- LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;
-};
+/**
+ * The metric definitions live in the Site Metrics module so the PDF report and the
+ * weekly email report the same numbers as this dashboard.
+ */
+export const buildOverviewQuery = (params: FilterParams, siteId: number) =>
+ buildOverviewMetricsQuery(buildMetricsSpec(params, siteId));
export interface OverviewRequest {
Params: {
@@ -69,7 +26,7 @@ export const getOverview = analyticsRoute<OverviewRequest>(
async (req: FastifyRequest<OverviewRequest>, res: FastifyReply) => {
const siteId = Number(req.params.siteId);
- const data = await runAnalyticsQuery<GetOverviewResponse>({
+ const data = await runAnalyticsQuery<OverviewRow>({
query: buildOverviewQuery(req.query, siteId),
params: { siteId },
});
diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.ts
index 23f95f1b3..f1e78f371 100644
--- a/server/src/api/analytics/utils/utils.ts
+++ b/server/src/api/analytics/utils/utils.ts
@@ -12,8 +12,14 @@ export const normalizeDatetimeForClickhouse = (value: string) => {
return new Date(withZone).toISOString().slice(0, 19).replace("T", " ");
};
-export function getTimeStatement(
- params: Pick<
+/**
+ * Every field is optional: a date range, a datetime range and a past-minutes
+ * window are three alternative ways to say the same thing, and callers outside
+ * HTTP (the report jobs) supply only one of them. No time params at all is a
+ * legitimate all-time query and yields an empty statement.
+ */
+type TimeParams = Partial<
+ Pick<
FilterParams,
| "start_date"
| "end_date"
@@ -23,7 +29,9 @@ export function getTimeStatement(
| "past_minutes_start"
| "past_minutes_end"
>
-) {
+>;
+
+export function getTimeStatement(params: TimeParams) {
const { start_date, end_date, time_zone, start_datetime, end_datetime, past_minutes_start, past_minutes_end } =
params;
diff --git a/server/src/services/pdfReports/pdfReportService.ts b/server/src/services/pdfReports/pdfReportService.ts
index b95c1d926..5bcb29b93 100644
--- a/server/src/services/pdfReports/pdfReportService.ts
+++ b/server/src/services/pdfReports/pdfReportService.ts
@@ -6,9 +6,14 @@ import { db } from "../../db/postgres/postgres.js";
import { sites } from "../../db/postgres/schema.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { getTimeStatement, processResults } from "../../api/analytics/utils/utils.js";
-import { effectiveUserId } from "../../api/analytics/utils/effectiveUserId.js";
-import { getFilterStatement } from "../../api/analytics/utils/getFilterStatement.js";
import { createServiceLogger } from "../../lib/logger/logger.js";
+import {
+ BreakdownDimension,
+ buildBreakdownQuery,
+ buildMetricsSpecForWindow,
+ buildOverviewQuery,
+ SiteMetricsSpec,
+} from "../siteMetrics/siteMetrics.js";
import { PdfReportTemplate } from "./templates/PdfReportTemplate.js";
import type { PdfReportParams, PdfReportData, OverviewData, MetricData, ChartDataPoint } from "./pdfReportTypes.js";
@@ -67,7 +72,14 @@ class PdfReportService {
time_zone: timeZone,
});
- const filterStatement = filters ? getFilterStatement(JSON.stringify(filters), siteId) : "";
+ // Each period gets its own filter statement: session-level filters compile to
+ // a subquery that is bounded by the time statement passed here, so reusing the
+ // current period's filters for the previous one would compare a correctly
+ // scoped window against a mis-scoped one and bias every growth percentage.
+ const filtersJson = filters ? JSON.stringify(filters) : "";
+ const spec = buildMetricsSpecForWindow(filtersJson, siteId, timeStatement);
+ const previousSpec = buildMetricsSpecForWindow(filtersJson, siteId, previousTimeStatement);
+ const filterStatement = spec.filterStatement ?? "";
// Determine bucket size based on date range
const bucket = durationDays <= 1 ? "hour" : durationDays <= 60 ? "day" : "week";
@@ -86,17 +98,17 @@ class PdfReportService {
topBrowsers,
topOperatingSystems,
] = await Promise.all([
- this.fetchOverviewData(siteId, timeStatement, filterStatement),
- this.fetchOverviewData(siteId, previousTimeStatement, filterStatement),
+ this.fetchOverviewData(siteId, spec),
+ this.fetchOverviewData(siteId, previousSpec),
this.fetchChartData(siteId, startDate, endDate, timeZone, bucket, filterStatement),
- this.fetchTopN(siteId, "country", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "region", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "city", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "pathname", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "referrer", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "device_type", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "browser", timeStatement, 13, filterStatement),
- this.fetchTopN(siteId, "operating_system", timeStatement, 13, filterStatement),
+ this.fetchTopN(siteId, "country", spec),
+ this.fetchTopN(siteId, "region", spec),
+ this.fetchTopN(siteId, "city", spec),
+ this.fetchTopN(siteId, "pathname", spec),
+ this.fetchTopN(siteId, "referrer", spec),
+ this.fetchTopN(siteId, "device_type", spec),
+ this.fetchTopN(siteId, "browser", spec),
+ this.fetchTopN(siteId, "operating_system", spec),
]);
if (!overview) {
@@ -168,60 +180,12 @@ class PdfReportService {
}
}
- private async fetchOverviewData(
- siteId: number,
- timeStatement: string,
- filterStatement: string
- ): Promise<OverviewData | null> {
+ private async fetchOverviewData(siteId: number, spec: SiteMetricsSpec): Promise<OverviewData | null> {
try {
- const query = `SELECT
- session_stats.sessions,
- session_stats.pages_per_session,
- session_stats.bounce_rate * 100 AS bounce_rate,
- session_stats.session_duration,
- page_stats.pageviews,
- page_stats.users
- FROM
- (
- SELECT
- COUNT() AS sessions,
- AVG(pages_in_session) AS pages_per_session,
- sumIf(1, pages_in_session = 1) / COUNT() AS bounce_rate,
- AVG(end_time - start_time) AS session_duration
- FROM
- (
- SELECT
- session_id,
- MIN(timestamp) AS start_time,
- MAX(timestamp) AS end_time,
- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session
- FROM events
- WHERE
- site_id = {siteId:Int32}
- ${timeStatement}
- ${filterStatement}
- GROUP BY session_id
- )
- ) AS session_stats
- CROSS JOIN
- (
- SELECT
- COUNT(*) AS pageviews,
- COUNT(DISTINCT ${effectiveUserId()}) AS users
- FROM events
- WHERE
- site_id = {siteId:Int32}
- ${timeStatement}
- AND type = 'pageview'
- ${filterStatement}
- ) AS page_stats`;
-
const result = await clickhouse.query({
- query,
+ query: buildOverviewQuery(spec),
format: "JSONEachRow",
- query_params: {
- siteId,
- },
+ query_params: { siteId },
});
const data = await processResults<OverviewData>(result);
@@ -234,228 +198,20 @@ class PdfReportService {
private async fetchTopN(
siteId: number,
- parameter: string,
- timeStatement: string,
- limit: number = 10,
- filterStatement: string
+ dimension: BreakdownDimension,
+ spec: SiteMetricsSpec,
+ limit: number = 13
): Promise<MetricData[]> {
try {
- let query = "";
-
- if (parameter === "country") {
- query = `
- WITH PageStats AS (
- SELECT
- country as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND country IS NOT NULL
- AND country <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "pathname") {
- query = `
- WITH EventTimes AS (
- SELECT
- session_id,
- pathname,
- timestamp,
- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND type = 'pageview'
- ${timeStatement}
- ${filterStatement}
- ),
- PageDurations AS (
- SELECT
- session_id,
- pathname,
- timestamp,
- next_timestamp,
- if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds
- FROM EventTimes
- ),
- PathStats AS (
- SELECT
- pathname,
- count() as visits,
- count(DISTINCT session_id) as unique_sessions
- FROM PageDurations
- GROUP BY pathname
- )
- SELECT
- pathname as value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PathStats
- ORDER BY unique_sessions DESC
- LIMIT {limit:Int32}`;
- } else if (parameter === "referrer") {
- query = `
- WITH PageStats AS (
- SELECT
- domainWithoutWWW(referrer) as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND domainWithoutWWW(referrer) IS NOT NULL
- AND domainWithoutWWW(referrer) <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "device_type") {
- query = `
- WITH PageStats AS (
- SELECT
- device_type as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND device_type IS NOT NULL
- AND device_type <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "browser") {
- query = `
- WITH PageStats AS (
- SELECT
- browser as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND browser IS NOT NULL
- AND browser <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "operating_system") {
- query = `
- WITH PageStats AS (
- SELECT
- operating_system as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND operating_system IS NOT NULL
- AND operating_system <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "region") {
- query = `
- WITH PageStats AS (
- SELECT
- region as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND region IS NOT NULL
- AND region <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "city") {
- query = `
- WITH PageStats AS (
- SELECT
- city as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND city IS NOT NULL
- AND city <> ''
- ${timeStatement}
- ${filterStatement}
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- }
-
const result = await clickhouse.query({
- query,
+ query: buildBreakdownQuery(dimension, spec),
format: "JSONEachRow",
- query_params: {
- siteId,
- limit,
- },
+ query_params: { siteId, limit },
});
return await processResults<MetricData>(result);
} catch (error) {
- this.logger.error({ err: error, siteId, parameter }, "Error fetching top N data");
+ this.logger.error({ err: error, siteId, dimension }, "Error fetching top N data");
return [];
}
}
diff --git a/server/src/services/pdfReports/pdfReportTypes.ts b/server/src/services/pdfReports/pdfReportTypes.ts
index f46a56968..cc7bc2eb9 100644
--- a/server/src/services/pdfReports/pdfReportTypes.ts
+++ b/server/src/services/pdfReports/pdfReportTypes.ts
@@ -1,19 +1,10 @@
import type { Filter } from "@rybbit/shared";
+import type { BreakdownRow, OverviewRow } from "../siteMetrics/siteMetrics.js";
-export interface OverviewData {
- sessions: number;
- pageviews: number;
- users: number;
- pages_per_session: number | null;
- bounce_rate: number | null;
- session_duration: number;
-}
-
-export interface MetricData {
- value: string;
- count: number;
- percentage: number | null;
-}
+// The report renders exactly what the Site Metrics module returns; naming the
+// rows here keeps the template's imports stable.
+export type OverviewData = OverviewRow;
+export type MetricData = BreakdownRow;
export interface ChartDataPoint {
time: string;
diff --git a/server/src/services/siteMetrics/siteMetrics.test.ts b/server/src/services/siteMetrics/siteMetrics.test.ts
new file mode 100644
index 000000000..27dc8e370
--- /dev/null
+++ b/server/src/services/siteMetrics/siteMetrics.test.ts
@@ -0,0 +1,190 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../../db/clickhouse/clickhouse.js", () => ({
+ clickhouse: { query: vi.fn() },
+}));
+vi.mock("../../db/postgres/postgres.js", () => ({
+ db: {},
+}));
+
+import { FilterParams } from "@rybbit/shared";
+import { getTimeStatement } from "../../api/analytics/utils/utils.js";
+import { buildOverviewQuery as buildDashboardOverviewQuery } from "../../api/analytics/getOverview.js";
+import {
+ BreakdownDimension,
+ buildBreakdownQuery,
+ buildMetricsSpec,
+ buildMetricsSpecForWindow,
+ buildOverviewQuery,
+ SiteMetricsSpec,
+} from "./siteMetrics.js";
+
+const SITE_ID = 1;
+
+/** How the dashboard route expresses its window: a date range plus a timezone. */
+const DASHBOARD_PARAMS = {
+ start_date: "2026-07-17",
+ end_date: "2026-07-23",
+ time_zone: "UTC",
+ filters: "",
+} as FilterParams;
+
+/** How the PDF report expresses its window: the same builder, no HTTP request. */
+const pdfSpec: SiteMetricsSpec = {
+ timeStatement: getTimeStatement({ start_date: "2026-07-17", end_date: "2026-07-23", time_zone: "UTC" }),
+ filterStatement: "",
+};
+
+/** How the weekly email expresses its window: an exact UTC datetime range. */
+const weeklySpec: SiteMetricsSpec = {
+ timeStatement: getTimeStatement({
+ start_datetime: "2026-07-17 00:00:00",
+ end_datetime: "2026-07-24 00:00:00",
+ }),
+};
+
+/** Blanks out the one axis that legitimately differs between the three surfaces. */
+const withoutTimeBound = (sql: string, spec: SiteMetricsSpec) =>
+ sql.replace(spec.timeStatement, "<TIME>").replace(spec.timeStatement, "<TIME>");
+
+describe("site metrics", () => {
+ describe("one definition across all three surfaces", () => {
+ it("gives the dashboard, the PDF report and the weekly email the same overview query", () => {
+ const dashboard = withoutTimeBound(buildDashboardOverviewQuery(DASHBOARD_PARAMS, SITE_ID), {
+ timeStatement: getTimeStatement(DASHBOARD_PARAMS),
+ });
+ const pdf = withoutTimeBound(buildOverviewQuery(pdfSpec), pdfSpec);
+ const weekly = withoutTimeBound(buildOverviewQuery(weeklySpec), weeklySpec);
+
+ expect(pdf).toBe(dashboard);
+ expect(weekly).toBe(dashboard);
+ expect(dashboard).not.toContain(getTimeStatement(DASHBOARD_PARAMS));
+ });
+
+ it("gives every surface the same breakdown query for a dimension", () => {
+ const pdf = withoutTimeBound(buildBreakdownQuery("country", pdfSpec), pdfSpec);
+ const weekly = withoutTimeBound(buildBreakdownQuery("country", weeklySpec), weeklySpec);
+
+ expect(weekly).toBe(pdf);
+ });
+ });
+
+ describe("buildOverviewQuery", () => {
+ it("counts users by identity, falling back to the device fingerprint", () => {
+ const sql = buildOverviewQuery(pdfSpec);
+
+ expect(sql).toContain("COUNT(DISTINCT f.effective_user_id) AS users");
+ expect(sql).toContain("anyIf(identified_user_id, identified_user_id != '')");
+ expect(sql).toContain("anyLast(user_id)");
+ });
+
+ it("resolves the identity once per session rather than per event", () => {
+ const sql = buildOverviewQuery(pdfSpec);
+
+ // The identity expression must sit inside the session-grouped CTE, so a
+ // visitor whose first pageview predates identify() still counts once.
+ const sessionCteBody = sql.split("FilteredSessionsWithStats AS (")[1].split("GROUP BY session_id")[0];
+ expect(sessionCteBody).toContain("anyIf(identified_user_id");
+ });
+
+ it("never aliases the identity back onto user_id", () => {
+ // Shadowing a column with an alias used inside its own expression is a
+ // cyclic-alias error in ClickHouse.
+ expect(buildOverviewQuery(pdfSpec)).not.toMatch(/anyLast\(user_id\)\) AS user_id/);
+ });
+
+ it("reports bounce rate as a percentage of the session's full pageview count", () => {
+ const sql = buildOverviewQuery({ timeStatement: "", filterStatement: "AND browser = 'Chrome'" });
+
+ // A session that saw three pages is not a bounce because a filter hid two
+ // of them, so the bounce test reads the unfiltered CTE.
+ expect(sql).toContain("sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate");
+ const unfilteredCte = sql.split("AllSessionPageviews AS (")[1].split(")")[0];
+ expect(unfilteredCte).not.toContain("browser = 'Chrome'");
+ });
+
+ it("applies the filter to the session, pageview and user counts", () => {
+ const sql = buildOverviewQuery({ timeStatement: "", filterStatement: "AND browser = 'Chrome'" });
+ const filteredCte = sql.split("FilteredSessionsWithStats AS (")[1];
+
+ expect(filteredCte).toContain("browser = 'Chrome'");
+ });
+ });
+
+ describe("buildBreakdownQuery", () => {
+ const DIMENSION_COLUMNS: Record<BreakdownDimension, string> = {
+ browser: "browser",
+ city: "city",
+ country: "country",
+ device_type: "device_type",
+ operating_system: "operating_system",
+ pathname: "pathname",
+ referrer: "domainWithoutWWW(referrer)",
+ region: "region",
+ };
+
+ it.each(Object.entries(DIMENSION_COLUMNS))("breaks down by %s", (dimension, column) => {
+ const sql = buildBreakdownQuery(dimension as BreakdownDimension, pdfSpec);
+
+ expect(sql).toContain(`${column} AS value`);
+ expect(sql).toContain("COUNT(DISTINCT session_id) AS unique_sessions");
+ expect(sql).toContain("LIMIT {limit:Int32}");
+ expect(sql).toContain(pdfSpec.timeStatement);
+ });
+
+ it("counts a page only where it was viewed", () => {
+ expect(buildBreakdownQuery("pathname", pdfSpec)).toContain("AND type = 'pageview'");
+ });
+
+ it("does not restrict session properties to pageview events", () => {
+ expect(buildBreakdownQuery("country", pdfSpec)).not.toContain("type = 'pageview'");
+ });
+
+ it("ranks ties deterministically so paged and repeated reports agree", () => {
+ expect(buildBreakdownQuery("country", pdfSpec)).toContain("ORDER BY count DESC, value ASC");
+ });
+
+ it("carries the caller's filters into the breakdown", () => {
+ const sql = buildBreakdownQuery("country", { timeStatement: "", filterStatement: "AND browser = 'Chrome'" });
+
+ expect(sql).toContain("browser = 'Chrome'");
+ });
+ });
+
+ describe("buildMetricsSpec", () => {
+ it("derives both the window and the filters from HTTP query params", () => {
+ const spec = buildMetricsSpec(
+ { ...DASHBOARD_PARAMS, filters: JSON.stringify([{ parameter: "browser", type: "equals", value: ["Chrome"] }]) },
+ SITE_ID
+ );
+
+ expect(spec.timeStatement).toContain("2026-07-17");
+ expect(spec.filterStatement).toContain("Chrome");
+ });
+
+ it("bounds a session-level filter's subquery to the spec's own window", () => {
+ // event_name filters compile to `session_id IN (SELECT … FROM events …)`.
+ // An unbounded subquery qualifies a session on an event fired outside the
+ // period being reported β€” and scans the Site's whole history to do it.
+ const eventFilter = JSON.stringify([{ parameter: "event_name", type: "equals", value: ["signup"] }]);
+ const spec = buildMetricsSpecForWindow(eventFilter, SITE_ID, pdfSpec.timeStatement);
+
+ expect(spec.filterStatement).toContain("session_id IN (");
+ expect(spec.filterStatement).toContain("2026-07-17");
+ });
+
+ it("scopes each period's filters to that period, so growth comparisons are fair", () => {
+ const eventFilter = JSON.stringify([{ parameter: "event_name", type: "equals", value: ["signup"] }]);
+ const current = buildMetricsSpecForWindow(eventFilter, SITE_ID, getTimeStatement(DASHBOARD_PARAMS));
+ const previous = buildMetricsSpecForWindow(
+ eventFilter,
+ SITE_ID,
+ getTimeStatement({ start_date: "2026-07-10", end_date: "2026-07-16", time_zone: "UTC" })
+ );
+
+ expect(current.filterStatement).toContain("2026-07-17");
+ expect(previous.filterStatement).toContain("2026-07-10");
+ expect(previous.filterStatement).not.toContain("2026-07-23");
+ });
+ });
+});
diff --git a/server/src/services/siteMetrics/siteMetrics.ts b/server/src/services/siteMetrics/siteMetrics.ts
new file mode 100644
index 000000000..8a65e2664
--- /dev/null
+++ b/server/src/services/siteMetrics/siteMetrics.ts
@@ -0,0 +1,197 @@
+import { FilterParams } from "@rybbit/shared";
+import { EFFECTIVE_SESSION_USER_ID } from "../../api/analytics/utils/effectiveUserId.js";
+import { getFilterStatement } from "../../api/analytics/utils/getFilterStatement.js";
+import { getTimeStatement } from "../../api/analytics/utils/utils.js";
+
+/**
+ * The definition of a Site's headline metrics β€” sessions, pageviews, users,
+ * pages per session, bounce rate, session duration β€” and of the single-dimension
+ * breakdowns that sit beside them.
+ *
+ * Three surfaces report these numbers: the dashboard overview, the PDF report and
+ * the weekly email. They used to carry three transcriptions of the same SQL, and
+ * they had already drifted: the reports computed bounce rate and pages per session
+ * against the *filtered* session's pageview count, while the dashboard used the
+ * session's full pageview count, so a filtered PDF disagreed with the dashboard it
+ * was exported from. The definition here is the dashboard's β€” the one users see
+ * first, and the one with test coverage.
+ *
+ * Everything below is a pure string builder: no ClickHouse client, no Fastify. The
+ * only axis that genuinely varies between callers is how the time window is
+ * expressed, so that arrives pre-built in the spec.
+ */
+
+/** A Site's headline metrics for one time window. */
+export interface OverviewRow {
+ sessions: number;
+ pageviews: number;
+ users: number;
+ pages_per_session: number | null;
+ bounce_rate: number | null;
+ session_duration: number;
+}
+
+/** One row of a single-dimension breakdown, ranked by session count. */
+export interface BreakdownRow {
+ value: string;
+ count: number;
+ percentage: number | null;
+}
+
+/**
+ * The window and filters a metrics query runs over.
+ *
+ * Both fields are SQL fragments that already begin with `AND` (or are empty), so
+ * they drop straight into a `WHERE`. Callers build the time statement however
+ * suits them β€” {@link buildMetricsSpec} from HTTP query params, or
+ * `getTimeStatement` directly from a datetime range β€” and bind `siteId` (and
+ * `limit`, for breakdowns) as query parameters at execution time.
+ */
+export interface SiteMetricsSpec {
+ timeStatement: string;
+ filterStatement?: string;
+}
+
+/**
+ * Pairs a set of filters with the window they apply to.
+ *
+ * The window is not optional here on purpose. Session-level filters
+ * (`event_name`, `channel`, entry/exit page) compile to a `session_id IN (SELECT
+ * … FROM events WHERE …)` subquery, and that subquery is bounded by the time
+ * statement handed to `getFilterStatement`. Omit it and the subquery scans the
+ * Site's entire history, so a session qualifies on an event fired years outside
+ * the period being reported β€” and the scan is unbounded to boot. Building the
+ * pair in one place is what stops a caller from scoping the two to different
+ * windows.
+ */
+export function buildMetricsSpecForWindow(filters: string, siteId: number, timeStatement: string): SiteMetricsSpec {
+ return {
+ timeStatement,
+ filterStatement: getFilterStatement(filters, siteId, timeStatement),
+ };
+}
+
+/** Builds a spec from the shared HTTP filter params (dashboard-style callers). */
+export function buildMetricsSpec(params: FilterParams, siteId: number): SiteMetricsSpec {
+ return buildMetricsSpecForWindow(params.filters, siteId, getTimeStatement(params));
+}
+
+/**
+ * Sessions, pageviews, users, pages per session, bounce rate and session duration
+ * for one window. Binds `{siteId:Int32}`.
+ *
+ * Pages per session and bounce rate deliberately read from `AllSessionPageviews`,
+ * which is *not* filtered: a session that landed on three pages is not a bounce
+ * just because a filter narrowed the view to one of them.
+ */
+export function buildOverviewQuery(spec: SiteMetricsSpec): string {
+ const { timeStatement, filterStatement = "" } = spec;
+
+ return `
+ WITH
+ AllSessionPageviews AS (
+ SELECT
+ session_id,
+ countIf(type = 'pageview') AS total_pageviews_in_session
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ ${timeStatement}
+ GROUP BY session_id
+ ),
+ FilteredSessionsWithStats AS (
+ SELECT
+ session_id,
+ -- Aliased away from \`user_id\` on purpose: an alias that shadows a column
+ -- referenced inside its own expression is a cyclic-alias error in ClickHouse.
+ ${EFFECTIVE_SESSION_USER_ID} AS effective_user_id,
+ MIN(timestamp) AS start_time,
+ MAX(timestamp) AS end_time,
+ countIf(type = 'pageview') AS filtered_pageviews
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ ${filterStatement}
+ ${timeStatement}
+ GROUP BY session_id
+ )
+ SELECT
+ COUNT() AS sessions,
+ AVG(asp.total_pageviews_in_session) AS pages_per_session,
+ sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,
+ AVG(f.end_time - f.start_time) AS session_duration,
+ SUM(f.filtered_pageviews) AS pageviews,
+ COUNT(DISTINCT f.effective_user_id) AS users
+ FROM FilteredSessionsWithStats f
+ LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;
+}
+
+/** The dimensions a report can be broken down by. */
+export type BreakdownDimension =
+ | "browser"
+ | "city"
+ | "country"
+ | "device_type"
+ | "operating_system"
+ | "pathname"
+ | "referrer"
+ | "region";
+
+interface DimensionDefinition {
+ /** The expression rows are grouped by and reported as `value`. */
+ expression: string;
+ /** An extra predicate the rows must satisfy, already prefixed with `AND`. */
+ restriction?: string;
+}
+
+const DIMENSIONS: Record<BreakdownDimension, DimensionDefinition> = {
+ browser: { expression: "browser" },
+ city: { expression: "city" },
+ country: { expression: "country" },
+ device_type: { expression: "device_type" },
+ operating_system: { expression: "operating_system" },
+ // A page counts where it was actually viewed; the other dimensions are session
+ // properties carried on every event, so only this one restricts by type.
+ pathname: { expression: "pathname", restriction: "AND type = 'pageview'" },
+ referrer: { expression: "domainWithoutWWW(referrer)" },
+ region: { expression: "region" },
+};
+
+/**
+ * The top values of one dimension, ranked by the number of sessions that touched
+ * them, with each value's share of the total. Binds `{siteId:Int32}` and
+ * `{limit:Int32}`.
+ *
+ * Empty values are dropped for every dimension, `pathname` included. The report
+ * copies this replaces guarded the other dimensions but not `pathname`, so a
+ * malformed pageview stored with `pathname = ''` earned a blank row in the table
+ * and a share of the percentages. `getMetric` has always dropped those rows; this
+ * makes the reports agree.
+ */
+export function buildBreakdownQuery(dimension: BreakdownDimension, spec: SiteMetricsSpec): string {
+ const { expression, restriction = "" } = DIMENSIONS[dimension];
+ const { timeStatement, filterStatement = "" } = spec;
+
+ return `
+ WITH BreakdownStats AS (
+ SELECT
+ ${expression} AS value,
+ COUNT(DISTINCT session_id) AS unique_sessions
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ AND ${expression} IS NOT NULL
+ AND ${expression} <> ''
+ ${restriction}
+ ${timeStatement}
+ ${filterStatement}
+ GROUP BY value
+ )
+ SELECT
+ value,
+ unique_sessions AS count,
+ round((unique_sessions / SUM(unique_sessions) OVER ()) * 100, 2) AS percentage
+ FROM BreakdownStats
+ ORDER BY count DESC, value ASC
+ LIMIT {limit:Int32}`;
+}
diff --git a/server/src/services/weekyReports/weeklyReportService.ts b/server/src/services/weekyReports/weeklyReportService.ts
index 3f61883e8..3e6be1d2f 100644
--- a/server/src/services/weekyReports/weeklyReportService.ts
+++ b/server/src/services/weekyReports/weeklyReportService.ts
@@ -4,12 +4,17 @@ import { eq } from "drizzle-orm";
import { db } from "../../db/postgres/postgres.js";
import { organization, member, user, sites } from "../../db/postgres/schema.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { processResults } from "../../api/analytics/utils/utils.js";
-import { effectiveUserId } from "../../api/analytics/utils/effectiveUserId.js";
+import { getTimeStatement, processResults } from "../../api/analytics/utils/utils.js";
import { createServiceLogger } from "../../lib/logger/logger.js";
import { sendWeeklyReportEmail } from "../../lib/email/email.js";
import { filterSitesByMemberAccess } from "../../lib/siteAccess.js";
import { IS_CLOUD } from "../../lib/const.js";
+import {
+ BreakdownDimension,
+ buildBreakdownQuery,
+ buildOverviewQuery,
+ SiteMetricsSpec,
+} from "../siteMetrics/siteMetrics.js";
import type { OverviewData, MetricData, SiteReport, OrganizationReport } from "./weeklyReportTypes.js";
const MAX_SITE_REPORTS_PER_ORG = 10;
@@ -20,61 +25,12 @@ class WeeklyReportService {
constructor() {}
- private async fetchOverviewData(siteId: number, startDate: string, endDate: string): Promise<OverviewData | null> {
+ private async fetchOverviewData(siteId: number, spec: SiteMetricsSpec): Promise<OverviewData | null> {
try {
- const query = `SELECT
- session_stats.sessions,
- session_stats.pages_per_session,
- session_stats.bounce_rate * 100 AS bounce_rate,
- session_stats.session_duration,
- page_stats.pageviews,
- page_stats.users
- FROM
- (
- -- Session-level metrics
- SELECT
- COUNT() AS sessions,
- AVG(pages_in_session) AS pages_per_session,
- sumIf(1, pages_in_session = 1) / COUNT() AS bounce_rate,
- AVG(end_time - start_time) AS session_duration
- FROM
- (
- -- One row per session
- SELECT
- session_id,
- MIN(timestamp) AS start_time,
- MAX(timestamp) AS end_time,
- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- GROUP BY session_id
- )
- ) AS session_stats
- CROSS JOIN
- (
- -- Page-level and user-level metrics
- SELECT
- COUNT(*) AS pageviews,
- COUNT(DISTINCT ${effectiveUserId()}) AS users
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- AND type = 'pageview'
- ) AS page_stats`;
-
const result = await clickhouse.query({
- query,
+ query: buildOverviewQuery(spec),
format: "JSONEachRow",
- query_params: {
- siteId,
- startDate,
- endDate,
- },
+ query_params: { siteId },
});
const data = await processResults<OverviewData>(result);
@@ -87,138 +43,20 @@ class WeeklyReportService {
private async fetchTopN(
siteId: number,
- parameter: string,
- startDate: string,
- endDate: string,
+ dimension: BreakdownDimension,
+ spec: SiteMetricsSpec,
limit: number = 5
): Promise<MetricData[]> {
try {
- let query = "";
-
- if (parameter === "country") {
- query = `
- WITH PageStats AS (
- SELECT
- country as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND country IS NOT NULL
- AND country <> ''
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "pathname") {
- query = `
- WITH EventTimes AS (
- SELECT
- session_id,
- pathname,
- timestamp,
- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND type = 'pageview'
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- ),
- PageDurations AS (
- SELECT
- session_id,
- pathname,
- timestamp,
- next_timestamp,
- if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds
- FROM EventTimes
- ),
- PathStats AS (
- SELECT
- pathname,
- count() as visits,
- count(DISTINCT session_id) as unique_sessions
- FROM PageDurations
- GROUP BY pathname
- )
- SELECT
- pathname as value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PathStats
- ORDER BY unique_sessions DESC
- LIMIT {limit:Int32}`;
- } else if (parameter === "referrer") {
- query = `
- WITH PageStats AS (
- SELECT
- domainWithoutWWW(referrer) as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND domainWithoutWWW(referrer) IS NOT NULL
- AND domainWithoutWWW(referrer) <> ''
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- } else if (parameter === "device_type") {
- query = `
- WITH PageStats AS (
- SELECT
- device_type as value,
- COUNT(distinct(session_id)) as unique_sessions,
- COUNT() as pageviews
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND device_type IS NOT NULL
- AND device_type <> ''
- AND timestamp >= toDateTime({startDate:String})
- AND timestamp < toDateTime({endDate:String})
- GROUP BY value
- )
- SELECT
- value,
- unique_sessions as count,
- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage
- FROM PageStats
- ORDER BY count desc
- LIMIT {limit:Int32}`;
- }
-
const result = await clickhouse.query({
- query,
+ query: buildBreakdownQuery(dimension, spec),
format: "JSONEachRow",
- query_params: {
- siteId,
- startDate,
- endDate,
- limit,
- },
+ query_params: { siteId, limit },
});
return await processResults<MetricData>(result);
} catch (error) {
- this.logger.error({ err: error, siteId, parameter }, "Error fetching top N data");
+ this.logger.error({ err: error, siteId, dimension }, "Error fetching top N data");
return [];
}
}
@@ -236,16 +74,28 @@ class WeeklyReportService {
const previousWeekEnd = currentWeekStart;
const previousWeekStart = currentWeekStart.minus({ days: 7 });
- // Format dates for ClickHouse (YYYY-MM-DD HH:mm:ss)
+ // Format dates for ClickHouse (YYYY-MM-DD HH:mm:ss). The windows above are
+ // UTC, and getTimeStatement anchors the comparison in UTC too β€” the bound
+ // `toDateTime({date:String})` this replaced was interpreted in the
+ // ClickHouse server's timezone, so a non-UTC server shifted the week.
const formatDate = (date: DateTime) => date.toFormat("yyyy-MM-dd HH:mm:ss");
+ const specFor = (from: DateTime, to: DateTime): SiteMetricsSpec => ({
+ timeStatement: getTimeStatement({
+ start_datetime: formatDate(from),
+ end_datetime: formatDate(to),
+ }),
+ });
+
+ const currentSpec = specFor(currentWeekStart, currentWeekEnd);
+ const previousSpec = specFor(previousWeekStart, previousWeekEnd);
const [currentWeek, previousWeek, topCountries, topPages, topReferrers, deviceBreakdown] = await Promise.all([
- this.fetchOverviewData(siteId, formatDate(currentWeekStart), formatDate(currentWeekEnd)),
- this.fetchOverviewData(siteId, formatDate(previousWeekStart), formatDate(previousWeekEnd)),
- this.fetchTopN(siteId, "country", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),
- this.fetchTopN(siteId, "pathname", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),
- this.fetchTopN(siteId, "referrer", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),
- this.fetchTopN(siteId, "device_type", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),
+ this.fetchOverviewData(siteId, currentSpec),
+ this.fetchOverviewData(siteId, previousSpec),
+ this.fetchTopN(siteId, "country", currentSpec),
+ this.fetchTopN(siteId, "pathname", currentSpec),
+ this.fetchTopN(siteId, "referrer", currentSpec),
+ this.fetchTopN(siteId, "device_type", currentSpec),
]);
if (!currentWeek) {
diff --git a/server/src/services/weekyReports/weeklyReportTypes.ts b/server/src/services/weekyReports/weeklyReportTypes.ts
index 4a943b452..e4216941d 100644
--- a/server/src/services/weekyReports/weeklyReportTypes.ts
+++ b/server/src/services/weekyReports/weeklyReportTypes.ts
@@ -1,17 +1,9 @@
-export interface OverviewData {
- sessions: number;
- pageviews: number;
- users: number;
- pages_per_session: number | null;
- bounce_rate: number | null;
- session_duration: number;
-}
+import type { BreakdownRow, OverviewRow } from "../siteMetrics/siteMetrics.js";
-export interface MetricData {
- value: string;
- count: number;
- percentage: number | null;
-}
+// The email renders exactly what the Site Metrics module returns; naming the
+// rows here keeps the template's imports stable.
+export type OverviewData = OverviewRow;
+export type MetricData = BreakdownRow;
export interface SiteReport {
siteId: number;