πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
refactor(analytics): one definition of the Site metrics, three surfaces that call it (#1115)
Goldflag8fc9917507e0
diff --git a/server/src/api/analytics/getOverview.test.ts b/server/src/api/analytics/getOverview.test.tsindex 6da6a5a7c..5a38bc18b 100644--- a/server/src/api/analytics/getOverview.test.ts+++ b/server/src/api/analytics/getOverview.test.ts@@ -8,7 +8,6 @@ vi.mock("../../db/postgres/postgres.js", () => ({}));import { FilterParams } from "@rybbit/shared";-import { buildOverviewQuery } from "./getOverview.js";import { buildOverviewBucketedQuery } from "./getOverviewBucketed.js";const SITE_ID = 1;@@ -23,32 +22,9 @@ const baseParams = (overrides: Partial<Record<string, unknown>> = {}) =>...overrides,}) as FilterParams & { bucket: "day" };+// The unbucketed overview is built by the Site Metrics module, which the PDF+// report and the weekly email share; its coverage lives in siteMetrics.test.ts.describe("unique user counting", () => {- describe("buildOverviewQuery", () => {- it("counts users by identity, falling back to the device fingerprint", () => {- const sql = buildOverviewQuery(baseParams(), SITE_ID);-- expect(sql).toContain("COUNT(DISTINCT f.effective_user_id) AS users");- expect(sql).toContain("anyIf(identified_user_id, identified_user_id != '')");- expect(sql).toContain("anyLast(user_id)");- });-- it("resolves the identity once per session rather than per event", () => {- const sql = buildOverviewQuery(baseParams(), SITE_ID);-- // The identity expression must sit inside the session-grouped CTE, so a- // visitor whose first pageview predates identify() still counts once.- const sessionCteBody = sql.split("FilteredSessionsWithStats AS (")[1].split("GROUP BY session_id")[0];- expect(sessionCteBody).toContain("anyIf(identified_user_id");- });-- it("never aliases the identity back onto user_id", () => {- // Shadowing a column with an alias used inside its own expression is a- // cyclic-alias error in ClickHouse.- expect(buildOverviewQuery(baseParams(), SITE_ID)).not.toMatch(/anyLast\(user_id\)\) AS user_id/);- });- });-describe("buildOverviewBucketedQuery", () => {it("counts users by identity, falling back to the device fingerprint", () => {const sql = buildOverviewBucketedQuery(baseParams(), SITE_ID);diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.tsindex 3bd1fdc81..8d0f804ab 100644--- a/server/src/api/analytics/getOverview.ts+++ b/server/src/api/analytics/getOverview.ts@@ -1,61 +1,18 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { getFilterStatement } from "./utils/getFilterStatement.js";-import { getTimeStatement } from "./utils/utils.js";+import {+ buildMetricsSpec,+ buildOverviewQuery as buildOverviewMetricsQuery,+ OverviewRow,+} from "../../services/siteMetrics/siteMetrics.js";import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";-import { EFFECTIVE_SESSION_USER_ID } from "./utils/effectiveUserId.js";-type GetOverviewResponse = {- sessions: number;- pageviews: number;- users: number;- pages_per_session: number;- bounce_rate: number;- session_duration: number;-};--export const buildOverviewQuery = (params: FilterParams, siteId: number) => {- const timeStatement = getTimeStatement(params);- const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);-- return `- WITH- AllSessionPageviews AS (- SELECT- session_id,- countIf(type = 'pageview') AS total_pageviews_in_session- FROM events- WHERE- site_id = {siteId:Int32}- ${timeStatement}- GROUP BY session_id- ),- FilteredSessionsWithStats AS (- SELECT- session_id,- -- Aliased away from \`user_id\` on purpose: an alias that shadows a column- -- referenced inside its own expression is a cyclic-alias error in ClickHouse.- ${EFFECTIVE_SESSION_USER_ID} AS effective_user_id,- MIN(timestamp) AS start_time,- MAX(timestamp) AS end_time,- countIf(type = 'pageview') AS filtered_pageviews- FROM events- WHERE- site_id = {siteId:Int32}- ${filterStatement}- ${timeStatement}- GROUP BY session_id- )- SELECT- COUNT() AS sessions,- AVG(asp.total_pageviews_in_session) AS pages_per_session,- sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,- AVG(f.end_time - f.start_time) AS session_duration,- SUM(f.filtered_pageviews) AS pageviews,- COUNT(DISTINCT f.effective_user_id) AS users- FROM FilteredSessionsWithStats f- LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;-};+/**+ * The metric definitions live in the Site Metrics module so the PDF report and the+ * weekly email report the same numbers as this dashboard.+ */+export const buildOverviewQuery = (params: FilterParams, siteId: number) =>+ buildOverviewMetricsQuery(buildMetricsSpec(params, siteId));export interface OverviewRequest {Params: {@@ -69,7 +26,7 @@ export const getOverview = analyticsRoute<OverviewRequest>(async (req: FastifyRequest<OverviewRequest>, res: FastifyReply) => {const siteId = Number(req.params.siteId);- const data = await runAnalyticsQuery<GetOverviewResponse>({+ const data = await runAnalyticsQuery<OverviewRow>({query: buildOverviewQuery(req.query, siteId),params: { siteId },});diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.tsindex 23f95f1b3..f1e78f371 100644--- a/server/src/api/analytics/utils/utils.ts+++ b/server/src/api/analytics/utils/utils.ts@@ -12,8 +12,14 @@ export const normalizeDatetimeForClickhouse = (value: string) => {return new Date(withZone).toISOString().slice(0, 19).replace("T", " ");};-export function getTimeStatement(- params: Pick<+/**+ * Every field is optional: a date range, a datetime range and a past-minutes+ * window are three alternative ways to say the same thing, and callers outside+ * HTTP (the report jobs) supply only one of them. No time params at all is a+ * legitimate all-time query and yields an empty statement.+ */+type TimeParams = Partial<+ Pick<FilterParams,| "start_date"| "end_date"@@ -23,7 +29,9 @@ export function getTimeStatement(| "past_minutes_start"| "past_minutes_end">-) {+>;++export function getTimeStatement(params: TimeParams) {const { start_date, end_date, time_zone, start_datetime, end_datetime, past_minutes_start, past_minutes_end } =params;diff --git a/server/src/services/pdfReports/pdfReportService.ts b/server/src/services/pdfReports/pdfReportService.tsindex b95c1d926..5bcb29b93 100644--- a/server/src/services/pdfReports/pdfReportService.ts+++ b/server/src/services/pdfReports/pdfReportService.ts@@ -6,9 +6,14 @@ import { db } from "../../db/postgres/postgres.js";import { sites } from "../../db/postgres/schema.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { getTimeStatement, processResults } from "../../api/analytics/utils/utils.js";-import { effectiveUserId } from "../../api/analytics/utils/effectiveUserId.js";-import { getFilterStatement } from "../../api/analytics/utils/getFilterStatement.js";import { createServiceLogger } from "../../lib/logger/logger.js";+import {+ BreakdownDimension,+ buildBreakdownQuery,+ buildMetricsSpecForWindow,+ buildOverviewQuery,+ SiteMetricsSpec,+} from "../siteMetrics/siteMetrics.js";import { PdfReportTemplate } from "./templates/PdfReportTemplate.js";import type { PdfReportParams, PdfReportData, OverviewData, MetricData, ChartDataPoint } from "./pdfReportTypes.js";@@ -67,7 +72,14 @@ class PdfReportService {time_zone: timeZone,});- const filterStatement = filters ? getFilterStatement(JSON.stringify(filters), siteId) : "";+ // Each period gets its own filter statement: session-level filters compile to+ // a subquery that is bounded by the time statement passed here, so reusing the+ // current period's filters for the previous one would compare a correctly+ // scoped window against a mis-scoped one and bias every growth percentage.+ const filtersJson = filters ? JSON.stringify(filters) : "";+ const spec = buildMetricsSpecForWindow(filtersJson, siteId, timeStatement);+ const previousSpec = buildMetricsSpecForWindow(filtersJson, siteId, previousTimeStatement);+ const filterStatement = spec.filterStatement ?? "";// Determine bucket size based on date rangeconst bucket = durationDays <= 1 ? "hour" : durationDays <= 60 ? "day" : "week";@@ -86,17 +98,17 @@ class PdfReportService {topBrowsers,topOperatingSystems,] = await Promise.all([- this.fetchOverviewData(siteId, timeStatement, filterStatement),- this.fetchOverviewData(siteId, previousTimeStatement, filterStatement),+ this.fetchOverviewData(siteId, spec),+ this.fetchOverviewData(siteId, previousSpec),this.fetchChartData(siteId, startDate, endDate, timeZone, bucket, filterStatement),- this.fetchTopN(siteId, "country", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "region", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "city", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "pathname", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "referrer", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "device_type", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "browser", timeStatement, 13, filterStatement),- this.fetchTopN(siteId, "operating_system", timeStatement, 13, filterStatement),+ this.fetchTopN(siteId, "country", spec),+ this.fetchTopN(siteId, "region", spec),+ this.fetchTopN(siteId, "city", spec),+ this.fetchTopN(siteId, "pathname", spec),+ this.fetchTopN(siteId, "referrer", spec),+ this.fetchTopN(siteId, "device_type", spec),+ this.fetchTopN(siteId, "browser", spec),+ this.fetchTopN(siteId, "operating_system", spec),]);if (!overview) {@@ -168,60 +180,12 @@ class PdfReportService {}}- private async fetchOverviewData(- siteId: number,- timeStatement: string,- filterStatement: string- ): Promise<OverviewData | null> {+ private async fetchOverviewData(siteId: number, spec: SiteMetricsSpec): Promise<OverviewData | null> {try {- const query = `SELECT- session_stats.sessions,- session_stats.pages_per_session,- session_stats.bounce_rate * 100 AS bounce_rate,- session_stats.session_duration,- page_stats.pageviews,- page_stats.users- FROM- (- SELECT- COUNT() AS sessions,- AVG(pages_in_session) AS pages_per_session,- sumIf(1, pages_in_session = 1) / COUNT() AS bounce_rate,- AVG(end_time - start_time) AS session_duration- FROM- (- SELECT- session_id,- MIN(timestamp) AS start_time,- MAX(timestamp) AS end_time,- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session- FROM events- WHERE- site_id = {siteId:Int32}- ${timeStatement}- ${filterStatement}- GROUP BY session_id- )- ) AS session_stats- CROSS JOIN- (- SELECT- COUNT(*) AS pageviews,- COUNT(DISTINCT ${effectiveUserId()}) AS users- FROM events- WHERE- site_id = {siteId:Int32}- ${timeStatement}- AND type = 'pageview'- ${filterStatement}- ) AS page_stats`;-const result = await clickhouse.query({- query,+ query: buildOverviewQuery(spec),format: "JSONEachRow",- query_params: {- siteId,- },+ query_params: { siteId },});const data = await processResults<OverviewData>(result);@@ -234,228 +198,20 @@ class PdfReportService {private async fetchTopN(siteId: number,- parameter: string,- timeStatement: string,- limit: number = 10,- filterStatement: string+ dimension: BreakdownDimension,+ spec: SiteMetricsSpec,+ limit: number = 13): Promise<MetricData[]> {try {- let query = "";-- if (parameter === "country") {- query = `- WITH PageStats AS (- SELECT- country as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND country IS NOT NULL- AND country <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "pathname") {- query = `- WITH EventTimes AS (- SELECT- session_id,- pathname,- timestamp,- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp- FROM events- WHERE- site_id = {siteId:Int32}- AND type = 'pageview'- ${timeStatement}- ${filterStatement}- ),- PageDurations AS (- SELECT- session_id,- pathname,- timestamp,- next_timestamp,- if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds- FROM EventTimes- ),- PathStats AS (- SELECT- pathname,- count() as visits,- count(DISTINCT session_id) as unique_sessions- FROM PageDurations- GROUP BY pathname- )- SELECT- pathname as value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PathStats- ORDER BY unique_sessions DESC- LIMIT {limit:Int32}`;- } else if (parameter === "referrer") {- query = `- WITH PageStats AS (- SELECT- domainWithoutWWW(referrer) as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND domainWithoutWWW(referrer) IS NOT NULL- AND domainWithoutWWW(referrer) <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "device_type") {- query = `- WITH PageStats AS (- SELECT- device_type as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND device_type IS NOT NULL- AND device_type <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "browser") {- query = `- WITH PageStats AS (- SELECT- browser as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND browser IS NOT NULL- AND browser <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "operating_system") {- query = `- WITH PageStats AS (- SELECT- operating_system as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND operating_system IS NOT NULL- AND operating_system <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "region") {- query = `- WITH PageStats AS (- SELECT- region as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND region IS NOT NULL- AND region <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "city") {- query = `- WITH PageStats AS (- SELECT- city as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND city IS NOT NULL- AND city <> ''- ${timeStatement}- ${filterStatement}- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- }-const result = await clickhouse.query({- query,+ query: buildBreakdownQuery(dimension, spec),format: "JSONEachRow",- query_params: {- siteId,- limit,- },+ query_params: { siteId, limit },});return await processResults<MetricData>(result);} catch (error) {- this.logger.error({ err: error, siteId, parameter }, "Error fetching top N data");+ this.logger.error({ err: error, siteId, dimension }, "Error fetching top N data");return [];}}diff --git a/server/src/services/pdfReports/pdfReportTypes.ts b/server/src/services/pdfReports/pdfReportTypes.tsindex f46a56968..cc7bc2eb9 100644--- a/server/src/services/pdfReports/pdfReportTypes.ts+++ b/server/src/services/pdfReports/pdfReportTypes.ts@@ -1,19 +1,10 @@import type { Filter } from "@rybbit/shared";+import type { BreakdownRow, OverviewRow } from "../siteMetrics/siteMetrics.js";-export interface OverviewData {- sessions: number;- pageviews: number;- users: number;- pages_per_session: number | null;- bounce_rate: number | null;- session_duration: number;-}--export interface MetricData {- value: string;- count: number;- percentage: number | null;-}+// The report renders exactly what the Site Metrics module returns; naming the+// rows here keeps the template's imports stable.+export type OverviewData = OverviewRow;+export type MetricData = BreakdownRow;export interface ChartDataPoint {time: string;diff --git a/server/src/services/siteMetrics/siteMetrics.test.ts b/server/src/services/siteMetrics/siteMetrics.test.tsnew file mode 100644index 000000000..27dc8e370--- /dev/null+++ b/server/src/services/siteMetrics/siteMetrics.test.ts@@ -0,0 +1,190 @@+import { describe, expect, it, vi } from "vitest";++vi.mock("../../db/clickhouse/clickhouse.js", () => ({+ clickhouse: { query: vi.fn() },+}));+vi.mock("../../db/postgres/postgres.js", () => ({+ db: {},+}));++import { FilterParams } from "@rybbit/shared";+import { getTimeStatement } from "../../api/analytics/utils/utils.js";+import { buildOverviewQuery as buildDashboardOverviewQuery } from "../../api/analytics/getOverview.js";+import {+ BreakdownDimension,+ buildBreakdownQuery,+ buildMetricsSpec,+ buildMetricsSpecForWindow,+ buildOverviewQuery,+ SiteMetricsSpec,+} from "./siteMetrics.js";++const SITE_ID = 1;++/** How the dashboard route expresses its window: a date range plus a timezone. */+const DASHBOARD_PARAMS = {+ start_date: "2026-07-17",+ end_date: "2026-07-23",+ time_zone: "UTC",+ filters: "",+} as FilterParams;++/** How the PDF report expresses its window: the same builder, no HTTP request. */+const pdfSpec: SiteMetricsSpec = {+ timeStatement: getTimeStatement({ start_date: "2026-07-17", end_date: "2026-07-23", time_zone: "UTC" }),+ filterStatement: "",+};++/** How the weekly email expresses its window: an exact UTC datetime range. */+const weeklySpec: SiteMetricsSpec = {+ timeStatement: getTimeStatement({+ start_datetime: "2026-07-17 00:00:00",+ end_datetime: "2026-07-24 00:00:00",+ }),+};++/** Blanks out the one axis that legitimately differs between the three surfaces. */+const withoutTimeBound = (sql: string, spec: SiteMetricsSpec) =>+ sql.replace(spec.timeStatement, "<TIME>").replace(spec.timeStatement, "<TIME>");++describe("site metrics", () => {+ describe("one definition across all three surfaces", () => {+ it("gives the dashboard, the PDF report and the weekly email the same overview query", () => {+ const dashboard = withoutTimeBound(buildDashboardOverviewQuery(DASHBOARD_PARAMS, SITE_ID), {+ timeStatement: getTimeStatement(DASHBOARD_PARAMS),+ });+ const pdf = withoutTimeBound(buildOverviewQuery(pdfSpec), pdfSpec);+ const weekly = withoutTimeBound(buildOverviewQuery(weeklySpec), weeklySpec);++ expect(pdf).toBe(dashboard);+ expect(weekly).toBe(dashboard);+ expect(dashboard).not.toContain(getTimeStatement(DASHBOARD_PARAMS));+ });++ it("gives every surface the same breakdown query for a dimension", () => {+ const pdf = withoutTimeBound(buildBreakdownQuery("country", pdfSpec), pdfSpec);+ const weekly = withoutTimeBound(buildBreakdownQuery("country", weeklySpec), weeklySpec);++ expect(weekly).toBe(pdf);+ });+ });++ describe("buildOverviewQuery", () => {+ it("counts users by identity, falling back to the device fingerprint", () => {+ const sql = buildOverviewQuery(pdfSpec);++ expect(sql).toContain("COUNT(DISTINCT f.effective_user_id) AS users");+ expect(sql).toContain("anyIf(identified_user_id, identified_user_id != '')");+ expect(sql).toContain("anyLast(user_id)");+ });++ it("resolves the identity once per session rather than per event", () => {+ const sql = buildOverviewQuery(pdfSpec);++ // The identity expression must sit inside the session-grouped CTE, so a+ // visitor whose first pageview predates identify() still counts once.+ const sessionCteBody = sql.split("FilteredSessionsWithStats AS (")[1].split("GROUP BY session_id")[0];+ expect(sessionCteBody).toContain("anyIf(identified_user_id");+ });++ it("never aliases the identity back onto user_id", () => {+ // Shadowing a column with an alias used inside its own expression is a+ // cyclic-alias error in ClickHouse.+ expect(buildOverviewQuery(pdfSpec)).not.toMatch(/anyLast\(user_id\)\) AS user_id/);+ });++ it("reports bounce rate as a percentage of the session's full pageview count", () => {+ const sql = buildOverviewQuery({ timeStatement: "", filterStatement: "AND browser = 'Chrome'" });++ // A session that saw three pages is not a bounce because a filter hid two+ // of them, so the bounce test reads the unfiltered CTE.+ expect(sql).toContain("sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate");+ const unfilteredCte = sql.split("AllSessionPageviews AS (")[1].split(")")[0];+ expect(unfilteredCte).not.toContain("browser = 'Chrome'");+ });++ it("applies the filter to the session, pageview and user counts", () => {+ const sql = buildOverviewQuery({ timeStatement: "", filterStatement: "AND browser = 'Chrome'" });+ const filteredCte = sql.split("FilteredSessionsWithStats AS (")[1];++ expect(filteredCte).toContain("browser = 'Chrome'");+ });+ });++ describe("buildBreakdownQuery", () => {+ const DIMENSION_COLUMNS: Record<BreakdownDimension, string> = {+ browser: "browser",+ city: "city",+ country: "country",+ device_type: "device_type",+ operating_system: "operating_system",+ pathname: "pathname",+ referrer: "domainWithoutWWW(referrer)",+ region: "region",+ };++ it.each(Object.entries(DIMENSION_COLUMNS))("breaks down by %s", (dimension, column) => {+ const sql = buildBreakdownQuery(dimension as BreakdownDimension, pdfSpec);++ expect(sql).toContain(`${column} AS value`);+ expect(sql).toContain("COUNT(DISTINCT session_id) AS unique_sessions");+ expect(sql).toContain("LIMIT {limit:Int32}");+ expect(sql).toContain(pdfSpec.timeStatement);+ });++ it("counts a page only where it was viewed", () => {+ expect(buildBreakdownQuery("pathname", pdfSpec)).toContain("AND type = 'pageview'");+ });++ it("does not restrict session properties to pageview events", () => {+ expect(buildBreakdownQuery("country", pdfSpec)).not.toContain("type = 'pageview'");+ });++ it("ranks ties deterministically so paged and repeated reports agree", () => {+ expect(buildBreakdownQuery("country", pdfSpec)).toContain("ORDER BY count DESC, value ASC");+ });++ it("carries the caller's filters into the breakdown", () => {+ const sql = buildBreakdownQuery("country", { timeStatement: "", filterStatement: "AND browser = 'Chrome'" });++ expect(sql).toContain("browser = 'Chrome'");+ });+ });++ describe("buildMetricsSpec", () => {+ it("derives both the window and the filters from HTTP query params", () => {+ const spec = buildMetricsSpec(+ { ...DASHBOARD_PARAMS, filters: JSON.stringify([{ parameter: "browser", type: "equals", value: ["Chrome"] }]) },+ SITE_ID+ );++ expect(spec.timeStatement).toContain("2026-07-17");+ expect(spec.filterStatement).toContain("Chrome");+ });++ it("bounds a session-level filter's subquery to the spec's own window", () => {+ // event_name filters compile to `session_id IN (SELECT β¦ FROM events β¦)`.+ // An unbounded subquery qualifies a session on an event fired outside the+ // period being reported β and scans the Site's whole history to do it.+ const eventFilter = JSON.stringify([{ parameter: "event_name", type: "equals", value: ["signup"] }]);+ const spec = buildMetricsSpecForWindow(eventFilter, SITE_ID, pdfSpec.timeStatement);++ expect(spec.filterStatement).toContain("session_id IN (");+ expect(spec.filterStatement).toContain("2026-07-17");+ });++ it("scopes each period's filters to that period, so growth comparisons are fair", () => {+ const eventFilter = JSON.stringify([{ parameter: "event_name", type: "equals", value: ["signup"] }]);+ const current = buildMetricsSpecForWindow(eventFilter, SITE_ID, getTimeStatement(DASHBOARD_PARAMS));+ const previous = buildMetricsSpecForWindow(+ eventFilter,+ SITE_ID,+ getTimeStatement({ start_date: "2026-07-10", end_date: "2026-07-16", time_zone: "UTC" })+ );++ expect(current.filterStatement).toContain("2026-07-17");+ expect(previous.filterStatement).toContain("2026-07-10");+ expect(previous.filterStatement).not.toContain("2026-07-23");+ });+ });+});diff --git a/server/src/services/siteMetrics/siteMetrics.ts b/server/src/services/siteMetrics/siteMetrics.tsnew file mode 100644index 000000000..8a65e2664--- /dev/null+++ b/server/src/services/siteMetrics/siteMetrics.ts@@ -0,0 +1,197 @@+import { FilterParams } from "@rybbit/shared";+import { EFFECTIVE_SESSION_USER_ID } from "../../api/analytics/utils/effectiveUserId.js";+import { getFilterStatement } from "../../api/analytics/utils/getFilterStatement.js";+import { getTimeStatement } from "../../api/analytics/utils/utils.js";++/**+ * The definition of a Site's headline metrics β sessions, pageviews, users,+ * pages per session, bounce rate, session duration β and of the single-dimension+ * breakdowns that sit beside them.+ *+ * Three surfaces report these numbers: the dashboard overview, the PDF report and+ * the weekly email. They used to carry three transcriptions of the same SQL, and+ * they had already drifted: the reports computed bounce rate and pages per session+ * against the *filtered* session's pageview count, while the dashboard used the+ * session's full pageview count, so a filtered PDF disagreed with the dashboard it+ * was exported from. The definition here is the dashboard's β the one users see+ * first, and the one with test coverage.+ *+ * Everything below is a pure string builder: no ClickHouse client, no Fastify. The+ * only axis that genuinely varies between callers is how the time window is+ * expressed, so that arrives pre-built in the spec.+ */++/** A Site's headline metrics for one time window. */+export interface OverviewRow {+ sessions: number;+ pageviews: number;+ users: number;+ pages_per_session: number | null;+ bounce_rate: number | null;+ session_duration: number;+}++/** One row of a single-dimension breakdown, ranked by session count. */+export interface BreakdownRow {+ value: string;+ count: number;+ percentage: number | null;+}++/**+ * The window and filters a metrics query runs over.+ *+ * Both fields are SQL fragments that already begin with `AND` (or are empty), so+ * they drop straight into a `WHERE`. Callers build the time statement however+ * suits them β {@link buildMetricsSpec} from HTTP query params, or+ * `getTimeStatement` directly from a datetime range β and bind `siteId` (and+ * `limit`, for breakdowns) as query parameters at execution time.+ */+export interface SiteMetricsSpec {+ timeStatement: string;+ filterStatement?: string;+}++/**+ * Pairs a set of filters with the window they apply to.+ *+ * The window is not optional here on purpose. Session-level filters+ * (`event_name`, `channel`, entry/exit page) compile to a `session_id IN (SELECT+ * β¦ FROM events WHERE β¦)` subquery, and that subquery is bounded by the time+ * statement handed to `getFilterStatement`. Omit it and the subquery scans the+ * Site's entire history, so a session qualifies on an event fired years outside+ * the period being reported β and the scan is unbounded to boot. Building the+ * pair in one place is what stops a caller from scoping the two to different+ * windows.+ */+export function buildMetricsSpecForWindow(filters: string, siteId: number, timeStatement: string): SiteMetricsSpec {+ return {+ timeStatement,+ filterStatement: getFilterStatement(filters, siteId, timeStatement),+ };+}++/** Builds a spec from the shared HTTP filter params (dashboard-style callers). */+export function buildMetricsSpec(params: FilterParams, siteId: number): SiteMetricsSpec {+ return buildMetricsSpecForWindow(params.filters, siteId, getTimeStatement(params));+}++/**+ * Sessions, pageviews, users, pages per session, bounce rate and session duration+ * for one window. Binds `{siteId:Int32}`.+ *+ * Pages per session and bounce rate deliberately read from `AllSessionPageviews`,+ * which is *not* filtered: a session that landed on three pages is not a bounce+ * just because a filter narrowed the view to one of them.+ */+export function buildOverviewQuery(spec: SiteMetricsSpec): string {+ const { timeStatement, filterStatement = "" } = spec;++ return `+ WITH+ AllSessionPageviews AS (+ SELECT+ session_id,+ countIf(type = 'pageview') AS total_pageviews_in_session+ FROM events+ WHERE+ site_id = {siteId:Int32}+ ${timeStatement}+ GROUP BY session_id+ ),+ FilteredSessionsWithStats AS (+ SELECT+ session_id,+ -- Aliased away from \`user_id\` on purpose: an alias that shadows a column+ -- referenced inside its own expression is a cyclic-alias error in ClickHouse.+ ${EFFECTIVE_SESSION_USER_ID} AS effective_user_id,+ MIN(timestamp) AS start_time,+ MAX(timestamp) AS end_time,+ countIf(type = 'pageview') AS filtered_pageviews+ FROM events+ WHERE+ site_id = {siteId:Int32}+ ${filterStatement}+ ${timeStatement}+ GROUP BY session_id+ )+ SELECT+ COUNT() AS sessions,+ AVG(asp.total_pageviews_in_session) AS pages_per_session,+ sumIf(1, asp.total_pageviews_in_session = 1) / COUNT() * 100 AS bounce_rate,+ AVG(f.end_time - f.start_time) AS session_duration,+ SUM(f.filtered_pageviews) AS pageviews,+ COUNT(DISTINCT f.effective_user_id) AS users+ FROM FilteredSessionsWithStats f+ LEFT JOIN AllSessionPageviews asp ON f.session_id = asp.session_id`;+}++/** The dimensions a report can be broken down by. */+export type BreakdownDimension =+ | "browser"+ | "city"+ | "country"+ | "device_type"+ | "operating_system"+ | "pathname"+ | "referrer"+ | "region";++interface DimensionDefinition {+ /** The expression rows are grouped by and reported as `value`. */+ expression: string;+ /** An extra predicate the rows must satisfy, already prefixed with `AND`. */+ restriction?: string;+}++const DIMENSIONS: Record<BreakdownDimension, DimensionDefinition> = {+ browser: { expression: "browser" },+ city: { expression: "city" },+ country: { expression: "country" },+ device_type: { expression: "device_type" },+ operating_system: { expression: "operating_system" },+ // A page counts where it was actually viewed; the other dimensions are session+ // properties carried on every event, so only this one restricts by type.+ pathname: { expression: "pathname", restriction: "AND type = 'pageview'" },+ referrer: { expression: "domainWithoutWWW(referrer)" },+ region: { expression: "region" },+};++/**+ * The top values of one dimension, ranked by the number of sessions that touched+ * them, with each value's share of the total. Binds `{siteId:Int32}` and+ * `{limit:Int32}`.+ *+ * Empty values are dropped for every dimension, `pathname` included. The report+ * copies this replaces guarded the other dimensions but not `pathname`, so a+ * malformed pageview stored with `pathname = ''` earned a blank row in the table+ * and a share of the percentages. `getMetric` has always dropped those rows; this+ * makes the reports agree.+ */+export function buildBreakdownQuery(dimension: BreakdownDimension, spec: SiteMetricsSpec): string {+ const { expression, restriction = "" } = DIMENSIONS[dimension];+ const { timeStatement, filterStatement = "" } = spec;++ return `+ WITH BreakdownStats AS (+ SELECT+ ${expression} AS value,+ COUNT(DISTINCT session_id) AS unique_sessions+ FROM events+ WHERE+ site_id = {siteId:Int32}+ AND ${expression} IS NOT NULL+ AND ${expression} <> ''+ ${restriction}+ ${timeStatement}+ ${filterStatement}+ GROUP BY value+ )+ SELECT+ value,+ unique_sessions AS count,+ round((unique_sessions / SUM(unique_sessions) OVER ()) * 100, 2) AS percentage+ FROM BreakdownStats+ ORDER BY count DESC, value ASC+ LIMIT {limit:Int32}`;+}diff --git a/server/src/services/weekyReports/weeklyReportService.ts b/server/src/services/weekyReports/weeklyReportService.tsindex 3f61883e8..3e6be1d2f 100644--- a/server/src/services/weekyReports/weeklyReportService.ts+++ b/server/src/services/weekyReports/weeklyReportService.ts@@ -4,12 +4,17 @@ import { eq } from "drizzle-orm";import { db } from "../../db/postgres/postgres.js";import { organization, member, user, sites } from "../../db/postgres/schema.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { processResults } from "../../api/analytics/utils/utils.js";-import { effectiveUserId } from "../../api/analytics/utils/effectiveUserId.js";+import { getTimeStatement, processResults } from "../../api/analytics/utils/utils.js";import { createServiceLogger } from "../../lib/logger/logger.js";import { sendWeeklyReportEmail } from "../../lib/email/email.js";import { filterSitesByMemberAccess } from "../../lib/siteAccess.js";import { IS_CLOUD } from "../../lib/const.js";+import {+ BreakdownDimension,+ buildBreakdownQuery,+ buildOverviewQuery,+ SiteMetricsSpec,+} from "../siteMetrics/siteMetrics.js";import type { OverviewData, MetricData, SiteReport, OrganizationReport } from "./weeklyReportTypes.js";const MAX_SITE_REPORTS_PER_ORG = 10;@@ -20,61 +25,12 @@ class WeeklyReportService {constructor() {}- private async fetchOverviewData(siteId: number, startDate: string, endDate: string): Promise<OverviewData | null> {+ private async fetchOverviewData(siteId: number, spec: SiteMetricsSpec): Promise<OverviewData | null> {try {- const query = `SELECT- session_stats.sessions,- session_stats.pages_per_session,- session_stats.bounce_rate * 100 AS bounce_rate,- session_stats.session_duration,- page_stats.pageviews,- page_stats.users- FROM- (- -- Session-level metrics- SELECT- COUNT() AS sessions,- AVG(pages_in_session) AS pages_per_session,- sumIf(1, pages_in_session = 1) / COUNT() AS bounce_rate,- AVG(end_time - start_time) AS session_duration- FROM- (- -- One row per session- SELECT- session_id,- MIN(timestamp) AS start_time,- MAX(timestamp) AS end_time,- COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session- FROM events- WHERE- site_id = {siteId:Int32}- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- GROUP BY session_id- )- ) AS session_stats- CROSS JOIN- (- -- Page-level and user-level metrics- SELECT- COUNT(*) AS pageviews,- COUNT(DISTINCT ${effectiveUserId()}) AS users- FROM events- WHERE- site_id = {siteId:Int32}- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- AND type = 'pageview'- ) AS page_stats`;-const result = await clickhouse.query({- query,+ query: buildOverviewQuery(spec),format: "JSONEachRow",- query_params: {- siteId,- startDate,- endDate,- },+ query_params: { siteId },});const data = await processResults<OverviewData>(result);@@ -87,138 +43,20 @@ class WeeklyReportService {private async fetchTopN(siteId: number,- parameter: string,- startDate: string,- endDate: string,+ dimension: BreakdownDimension,+ spec: SiteMetricsSpec,limit: number = 5): Promise<MetricData[]> {try {- let query = "";-- if (parameter === "country") {- query = `- WITH PageStats AS (- SELECT- country as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND country IS NOT NULL- AND country <> ''- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "pathname") {- query = `- WITH EventTimes AS (- SELECT- session_id,- pathname,- timestamp,- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp- FROM events- WHERE- site_id = {siteId:Int32}- AND type = 'pageview'- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- ),- PageDurations AS (- SELECT- session_id,- pathname,- timestamp,- next_timestamp,- if(isNull(next_timestamp), 0, dateDiff('second', timestamp, next_timestamp)) as time_diff_seconds- FROM EventTimes- ),- PathStats AS (- SELECT- pathname,- count() as visits,- count(DISTINCT session_id) as unique_sessions- FROM PageDurations- GROUP BY pathname- )- SELECT- pathname as value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PathStats- ORDER BY unique_sessions DESC- LIMIT {limit:Int32}`;- } else if (parameter === "referrer") {- query = `- WITH PageStats AS (- SELECT- domainWithoutWWW(referrer) as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND domainWithoutWWW(referrer) IS NOT NULL- AND domainWithoutWWW(referrer) <> ''- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- } else if (parameter === "device_type") {- query = `- WITH PageStats AS (- SELECT- device_type as value,- COUNT(distinct(session_id)) as unique_sessions,- COUNT() as pageviews- FROM events- WHERE- site_id = {siteId:Int32}- AND device_type IS NOT NULL- AND device_type <> ''- AND timestamp >= toDateTime({startDate:String})- AND timestamp < toDateTime({endDate:String})- GROUP BY value- )- SELECT- value,- unique_sessions as count,- round((unique_sessions / sum(unique_sessions) OVER ()) * 100, 2) as percentage- FROM PageStats- ORDER BY count desc- LIMIT {limit:Int32}`;- }-const result = await clickhouse.query({- query,+ query: buildBreakdownQuery(dimension, spec),format: "JSONEachRow",- query_params: {- siteId,- startDate,- endDate,- limit,- },+ query_params: { siteId, limit },});return await processResults<MetricData>(result);} catch (error) {- this.logger.error({ err: error, siteId, parameter }, "Error fetching top N data");+ this.logger.error({ err: error, siteId, dimension }, "Error fetching top N data");return [];}}@@ -236,16 +74,28 @@ class WeeklyReportService {const previousWeekEnd = currentWeekStart;const previousWeekStart = currentWeekStart.minus({ days: 7 });- // Format dates for ClickHouse (YYYY-MM-DD HH:mm:ss)+ // Format dates for ClickHouse (YYYY-MM-DD HH:mm:ss). The windows above are+ // UTC, and getTimeStatement anchors the comparison in UTC too β the bound+ // `toDateTime({date:String})` this replaced was interpreted in the+ // ClickHouse server's timezone, so a non-UTC server shifted the week.const formatDate = (date: DateTime) => date.toFormat("yyyy-MM-dd HH:mm:ss");+ const specFor = (from: DateTime, to: DateTime): SiteMetricsSpec => ({+ timeStatement: getTimeStatement({+ start_datetime: formatDate(from),+ end_datetime: formatDate(to),+ }),+ });++ const currentSpec = specFor(currentWeekStart, currentWeekEnd);+ const previousSpec = specFor(previousWeekStart, previousWeekEnd);const [currentWeek, previousWeek, topCountries, topPages, topReferrers, deviceBreakdown] = await Promise.all([- this.fetchOverviewData(siteId, formatDate(currentWeekStart), formatDate(currentWeekEnd)),- this.fetchOverviewData(siteId, formatDate(previousWeekStart), formatDate(previousWeekEnd)),- this.fetchTopN(siteId, "country", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),- this.fetchTopN(siteId, "pathname", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),- this.fetchTopN(siteId, "referrer", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),- this.fetchTopN(siteId, "device_type", formatDate(currentWeekStart), formatDate(currentWeekEnd), 5),+ this.fetchOverviewData(siteId, currentSpec),+ this.fetchOverviewData(siteId, previousSpec),+ this.fetchTopN(siteId, "country", currentSpec),+ this.fetchTopN(siteId, "pathname", currentSpec),+ this.fetchTopN(siteId, "referrer", currentSpec),+ this.fetchTopN(siteId, "device_type", currentSpec),]);if (!currentWeek) {diff --git a/server/src/services/weekyReports/weeklyReportTypes.ts b/server/src/services/weekyReports/weeklyReportTypes.tsindex 4a943b452..e4216941d 100644--- a/server/src/services/weekyReports/weeklyReportTypes.ts+++ b/server/src/services/weekyReports/weeklyReportTypes.ts@@ -1,17 +1,9 @@-export interface OverviewData {- sessions: number;- pageviews: number;- users: number;- pages_per_session: number | null;- bounce_rate: number | null;- session_duration: number;-}+import type { BreakdownRow, OverviewRow } from "../siteMetrics/siteMetrics.js";-export interface MetricData {- value: string;- count: number;- percentage: number | null;-}+// The email renders exactly what the Site Metrics module returns; naming the+// rows here keeps the template's imports stable.+export type OverviewData = OverviewRow;+export type MetricData = BreakdownRow;export interface SiteReport {siteId: number;