🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor session attribution logic in analytics API
Bill Yangcd45535c5049
diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.tsindex 09bda0f6e..6e26ec4c1 100644--- a/server/src/api/analytics/getMetric.ts+++ b/server/src/api/analytics/getMetric.ts@@ -3,6 +3,7 @@ import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { FilterParameter } from "./types.js";import { getTimeStatement, processResults } from "./utils/utils.js";import { getFilterStatement, getSqlParam } from "./utils/getFilterStatement.js";+import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";import { FilterParams } from "@rybbit/shared";interface GetMetricRequest {@@ -353,12 +354,16 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole// Default case for other parametersconst sqlParam = getSqlParam(parameter);+ // Sessions are attributed to their first attributed channel (matching the+ // sessions views), not the first event's channel, which is often 'Direct'.+ const valueExpression = parameter === "channel" ? SESSION_CHANNEL_AGG : `argMin(${sqlParam}, e.timestamp)`;+if (isCountQuery) {return `SELECT COUNT(DISTINCT value) as totalCountFROM (SELECT- argMin(${sqlParam}, e.timestamp) as value+ ${valueExpression} as valueFROM events eWHEREe.site_id = {siteId:Int32}@@ -385,7 +390,7 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole),SessionData AS (SELECT- argMin(${sqlParam}, e.timestamp) as value,+ ${valueExpression} as value,e.session_id,any(spc.pageviews_in_session) as pageviews_in_sessionFROM events ediff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.tsindex e0d115b29..08d532873 100644--- a/server/src/api/analytics/sessions/getSession.ts+++ b/server/src/api/analytics/sessions/getSession.ts@@ -1,5 +1,6 @@import { FastifyReply, FastifyRequest } from "fastify";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";import { processResults } from "../utils/utils.js";export interface SessionDetails {@@ -92,8 +93,8 @@ SELECTany(operating_system_version) as operating_system_version,any(screen_width) as screen_width,any(screen_height) as screen_height,- argMin(referrer, timestamp) as referrer,- argMin(channel, timestamp) as channel,+ ${SESSION_REFERRER_AGG} as referrer,+ ${SESSION_CHANNEL_AGG} as channel,min(timestamp) as session_start,max(timestamp) as session_end,dateDiff('second', min(timestamp), max(timestamp)) as session_duration,diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.tsindex ccb1bc506..e53a8d0bb 100644--- a/server/src/api/analytics/sessions/getSessions.ts+++ b/server/src/api/analytics/sessions/getSessions.ts@@ -2,6 +2,7 @@ import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";export type GetSessionsResponse = {@@ -120,8 +121,8 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:argMax(operating_system_version, timestamp) AS operating_system_version,argMax(screen_width, timestamp) AS screen_width,argMax(screen_height, timestamp) AS screen_height,- argMin(referrer, timestamp) AS referrer,- argMin(channel, timestamp) AS channel,+ ${SESSION_REFERRER_AGG} AS referrer,+ ${SESSION_CHANNEL_AGG} AS channel,argMin(hostname, timestamp) AS hostname,argMin(url_parameters, timestamp)['utm_source'] AS utm_source,argMin(url_parameters, timestamp)['utm_medium'] AS utm_medium,diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.tsindex abcb32b39..9189dd257 100644--- a/server/src/api/analytics/users/getUserInfo.ts+++ b/server/src/api/analytics/users/getUserInfo.ts@@ -3,6 +3,7 @@ import { eq, and } from "drizzle-orm";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { db } from "../../../db/postgres/postgres.js";import { userProfiles, userAliases } from "../../../db/postgres/schema.js";+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";import { processResults } from "../utils/utils.js";interface UserPageviewData {@@ -73,8 +74,8 @@ export async function getUserInfo(argMax(operating_system_version, timestamp) AS operating_system_version,argMax(screen_width, timestamp) AS screen_width,argMax(screen_height, timestamp) AS screen_height,- argMin(referrer, timestamp) AS referrer,- argMin(channel, timestamp) AS channel,+ ${SESSION_REFERRER_AGG} AS referrer,+ ${SESSION_CHANNEL_AGG} AS channel,MAX(timestamp) AS session_end,MIN(timestamp) AS session_start,dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,diff --git a/server/src/api/analytics/users/getUsers.ts b/server/src/api/analytics/users/getUsers.tsindex 229d15700..587b272e4 100644--- a/server/src/api/analytics/users/getUsers.ts+++ b/server/src/api/analytics/users/getUsers.ts@@ -5,6 +5,7 @@ import { db } from "../../../db/postgres/postgres.js";import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";export type GetUsersResponse = {user_id: string; // Device fingerprint@@ -118,8 +119,8 @@ WITH AggregatedUsers AS (argMax(device_type, timestamp) AS device_type,argMax(screen_width, timestamp) AS screen_width,argMax(screen_height, timestamp) AS screen_height,- argMin(referrer, timestamp) AS referrer,- argMin(channel, timestamp) AS channel,+ ${SESSION_REFERRER_AGG} AS referrer,+ ${SESSION_CHANNEL_AGG} AS channel,argMin(hostname, timestamp) AS hostname,countIf(type = 'pageview') AS pageviews,countIf(type = 'custom_event') AS events,diff --git a/server/src/api/analytics/utils/getFilterStatement.test.ts b/server/src/api/analytics/utils/getFilterStatement.test.tsindex 881364c4c..0ef730836 100644--- a/server/src/api/analytics/utils/getFilterStatement.test.ts+++ b/server/src/api/analytics/utils/getFilterStatement.test.ts@@ -284,16 +284,15 @@ describe("getFilterStatement", () => {});describe("Channel session acquisition filter", () => {- it("should filter by the first non-empty channel in each session", () => {+ it("should filter by the first attributed channel in each session", () => {const filters = JSON.stringify([{ parameter: "channel", type: "equals", value: ["Organic Search"] }]);const result = getFilterStatement(filters, 123, "AND timestamp > now() - INTERVAL 1 DAY");expect(result).toContain("session_id IN");- expect(result).toContain("argMin(channel, timestamp) AS session_channel");+ expect(result).toContain("argMinIf(channel, timestamp, channel NOT IN ('Direct', 'Internal', ''))");+ expect(result).toContain("AS session_channel");expect(result).toContain("site_id = 123");expect(result).toContain("timestamp > now() - INTERVAL 1 DAY");- expect(result).toContain("channel IS NOT NULL");- expect(result).toContain("channel <> ''");expect(result).toContain("session_channel = 'Organic Search'");});diff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.tsindex fc5aefb23..672041a20 100644--- a/server/src/api/analytics/utils/getFilterStatement.ts+++ b/server/src/api/analytics/utils/getFilterStatement.ts@@ -1,12 +1,14 @@import SqlString from "sqlstring";import { filterParamSchema, validateFilters } from "./query-validation.js";+import { SESSION_CHANNEL_AGG } from "./sessionAttribution.js";import { FilterParameter, FilterType } from "../types.js";// Options for customizing filter behaviorexport interface FilterStatementOptions {// Parameters that should use session-level subqueries (finds sessions containing matching events)// Default: ["event_name", "channel"] - entry_page and exit_page are always session-level due to special aggregation- // Channel is handled as a session acquisition field using the first non-empty channel in the session.+ // Channel is handled as a session acquisition field using the session's first attributed channel+ // (SESSION_CHANNEL_AGG), matching how session views derive their channel.sessionLevelParams?: FilterParameter[];// Field name mappings for CTEs that extract fields to different column names@@ -186,26 +188,23 @@ export function getFilterStatement()`;};- const buildSessionFirstValueSubquery = (- param: FilterParameter,- alias: string,+ const buildSessionChannelSubquery = (filterType: FilterType,values: (string | number)[],wildcardPrefix: string): string => {- const whereClause = [siteIdFilter, timeFilter, `${param} IS NOT NULL`, `${param} <> ''`]- .filter(Boolean)- .join(" AND ");- const condition = buildStringFilterCondition(alias, filterType, values, wildcardPrefix);+ const whereClause = [siteIdFilter, timeFilter].filter(Boolean).join(" AND ");+ const whereStatement = whereClause ? `WHERE ${whereClause}` : "";+ const condition = buildStringFilterCondition("session_channel", filterType, values, wildcardPrefix);return `session_id IN (SELECT session_idFROM (SELECTsession_id,- argMin(${param}, timestamp) AS ${alias}+ ${SESSION_CHANNEL_AGG} AS session_channelFROM events- WHERE ${whereClause}+ ${whereStatement}GROUP BY session_id)WHERE ${condition}@@ -221,10 +220,10 @@ export function getFilterStatement(const isNullCheck = filter.type === "is_null" || filter.type === "is_not_null";// Handle session-level filters (configurable via options).- // Most parameters match sessions containing an event; channel uses the session's first value.+ // Most parameters match sessions containing an event; channel uses the session's first attributed value.if (sessionLevelParams.includes(filter.parameter)) {if (filter.parameter === "channel") {- return buildSessionFirstValueSubquery(filter.parameter, "session_channel", filter.type, filter.value, x);+ return buildSessionChannelSubquery(filter.type, filter.value, x);}return buildSessionLevelSubquery(filter.parameter, filter.type, filter.value, x);diff --git a/server/src/api/analytics/utils/sessionAttribution.ts b/server/src/api/analytics/utils/sessionAttribution.tsnew file mode 100644index 000000000..79f918348--- /dev/null+++ b/server/src/api/analytics/utils/sessionAttribution.ts@@ -0,0 +1,20 @@+// Session-level attribution aggregations, shared by every query that derives a+// session's (or user's) referrer/channel from its events. Keep these in sync with+// the per-parameter reports in getMetric.ts: the referrers report counts a session+// under its first non-empty referrer, so session views must derive referrer the+// same way or the same session shows different sources in different views.++// First non-empty referrer in the session; '' when every event is direct.+// Plain argMin(referrer, timestamp) is wrong here: a session that starts direct+// and later returns via an external link (30-minute session window) would show+// as Direct while the referrers report attributes it to that link.+export const SESSION_REFERRER_AGG = "argMinIf(referrer, timestamp, referrer != '')";++// Channel values that carry no acquisition signal: mid-session pageviews are+// 'Direct' (self-referrer cleared before getChannel) or 'Internal' (same-site+// referrer that survived clearing, e.g. www variants). '' guards legacy rows.+const UNATTRIBUTED_CHANNELS = "('Direct', 'Internal', '')";++// First attributed channel in the session, falling back to the first event's+// channel when no event in the session carries an acquisition signal.+export const SESSION_CHANNEL_AGG = `if(countIf(channel NOT IN ${UNATTRIBUTED_CHANNELS}) > 0, argMinIf(channel, timestamp, channel NOT IN ${UNATTRIBUTED_CHANNELS}), argMin(channel, timestamp))`;