Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor session attribution logic in analytics API

Bill Yangcd45535c5049
diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.ts
index 09bda0f6e..6e26ec4c1 100644
--- a/server/src/api/analytics/getMetric.ts
+++ b/server/src/api/analytics/getMetric.ts
@@ -3,6 +3,7 @@ import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { FilterParameter } from "./types.js";
import { getTimeStatement, processResults } from "./utils/utils.js";
import { getFilterStatement, getSqlParam } from "./utils/getFilterStatement.js";
+import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";
import { FilterParams } from "@rybbit/shared";
interface GetMetricRequest {
@@ -353,12 +354,16 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole
// Default case for other parameters
const sqlParam = getSqlParam(parameter);
+ // Sessions are attributed to their first attributed channel (matching the
+ // sessions views), not the first event's channel, which is often 'Direct'.
+ const valueExpression = parameter === "channel" ? SESSION_CHANNEL_AGG : `argMin(${sqlParam}, e.timestamp)`;
+
if (isCountQuery) {
return `
SELECT COUNT(DISTINCT value) as totalCount
FROM (
SELECT
- argMin(${sqlParam}, e.timestamp) as value
+ ${valueExpression} as value
FROM events e
WHERE
e.site_id = {siteId:Int32}
@@ -385,7 +390,7 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole
),
SessionData AS (
SELECT
- argMin(${sqlParam}, e.timestamp) as value,
+ ${valueExpression} as value,
e.session_id,
any(spc.pageviews_in_session) as pageviews_in_session
FROM events e
diff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.ts
index e0d115b29..08d532873 100644
--- a/server/src/api/analytics/sessions/getSession.ts
+++ b/server/src/api/analytics/sessions/getSession.ts
@@ -1,5 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
import { processResults } from "../utils/utils.js";
export interface SessionDetails {
@@ -92,8 +93,8 @@ SELECT
any(operating_system_version) as operating_system_version,
any(screen_width) as screen_width,
any(screen_height) as screen_height,
- argMin(referrer, timestamp) as referrer,
- argMin(channel, timestamp) as channel,
+ ${SESSION_REFERRER_AGG} as referrer,
+ ${SESSION_CHANNEL_AGG} as channel,
min(timestamp) as session_start,
max(timestamp) as session_end,
dateDiff('second', min(timestamp), max(timestamp)) as session_duration,
diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.ts
index ccb1bc506..e53a8d0bb 100644
--- a/server/src/api/analytics/sessions/getSessions.ts
+++ b/server/src/api/analytics/sessions/getSessions.ts
@@ -2,6 +2,7 @@ import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
export type GetSessionsResponse = {
@@ -120,8 +121,8 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:
argMax(operating_system_version, timestamp) AS operating_system_version,
argMax(screen_width, timestamp) AS screen_width,
argMax(screen_height, timestamp) AS screen_height,
- argMin(referrer, timestamp) AS referrer,
- argMin(channel, timestamp) AS channel,
+ ${SESSION_REFERRER_AGG} AS referrer,
+ ${SESSION_CHANNEL_AGG} AS channel,
argMin(hostname, timestamp) AS hostname,
argMin(url_parameters, timestamp)['utm_source'] AS utm_source,
argMin(url_parameters, timestamp)['utm_medium'] AS utm_medium,
diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.ts
index abcb32b39..9189dd257 100644
--- a/server/src/api/analytics/users/getUserInfo.ts
+++ b/server/src/api/analytics/users/getUserInfo.ts
@@ -3,6 +3,7 @@ import { eq, and } from "drizzle-orm";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { db } from "../../../db/postgres/postgres.js";
import { userProfiles, userAliases } from "../../../db/postgres/schema.js";
+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
import { processResults } from "../utils/utils.js";
interface UserPageviewData {
@@ -73,8 +74,8 @@ export async function getUserInfo(
argMax(operating_system_version, timestamp) AS operating_system_version,
argMax(screen_width, timestamp) AS screen_width,
argMax(screen_height, timestamp) AS screen_height,
- argMin(referrer, timestamp) AS referrer,
- argMin(channel, timestamp) AS channel,
+ ${SESSION_REFERRER_AGG} AS referrer,
+ ${SESSION_CHANNEL_AGG} AS channel,
MAX(timestamp) AS session_end,
MIN(timestamp) AS session_start,
dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,
diff --git a/server/src/api/analytics/users/getUsers.ts b/server/src/api/analytics/users/getUsers.ts
index 229d15700..587b272e4 100644
--- a/server/src/api/analytics/users/getUsers.ts
+++ b/server/src/api/analytics/users/getUsers.ts
@@ -5,6 +5,7 @@ import { db } from "../../../db/postgres/postgres.js";
import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
export type GetUsersResponse = {
user_id: string; // Device fingerprint
@@ -118,8 +119,8 @@ WITH AggregatedUsers AS (
argMax(device_type, timestamp) AS device_type,
argMax(screen_width, timestamp) AS screen_width,
argMax(screen_height, timestamp) AS screen_height,
- argMin(referrer, timestamp) AS referrer,
- argMin(channel, timestamp) AS channel,
+ ${SESSION_REFERRER_AGG} AS referrer,
+ ${SESSION_CHANNEL_AGG} AS channel,
argMin(hostname, timestamp) AS hostname,
countIf(type = 'pageview') AS pageviews,
countIf(type = 'custom_event') AS events,
diff --git a/server/src/api/analytics/utils/getFilterStatement.test.ts b/server/src/api/analytics/utils/getFilterStatement.test.ts
index 881364c4c..0ef730836 100644
--- a/server/src/api/analytics/utils/getFilterStatement.test.ts
+++ b/server/src/api/analytics/utils/getFilterStatement.test.ts
@@ -284,16 +284,15 @@ describe("getFilterStatement", () => {
});
describe("Channel session acquisition filter", () => {
- it("should filter by the first non-empty channel in each session", () => {
+ it("should filter by the first attributed channel in each session", () => {
const filters = JSON.stringify([{ parameter: "channel", type: "equals", value: ["Organic Search"] }]);
const result = getFilterStatement(filters, 123, "AND timestamp > now() - INTERVAL 1 DAY");
expect(result).toContain("session_id IN");
- expect(result).toContain("argMin(channel, timestamp) AS session_channel");
+ expect(result).toContain("argMinIf(channel, timestamp, channel NOT IN ('Direct', 'Internal', ''))");
+ expect(result).toContain("AS session_channel");
expect(result).toContain("site_id = 123");
expect(result).toContain("timestamp > now() - INTERVAL 1 DAY");
- expect(result).toContain("channel IS NOT NULL");
- expect(result).toContain("channel <> ''");
expect(result).toContain("session_channel = 'Organic Search'");
});
diff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.ts
index fc5aefb23..672041a20 100644
--- a/server/src/api/analytics/utils/getFilterStatement.ts
+++ b/server/src/api/analytics/utils/getFilterStatement.ts
@@ -1,12 +1,14 @@
import SqlString from "sqlstring";
import { filterParamSchema, validateFilters } from "./query-validation.js";
+import { SESSION_CHANNEL_AGG } from "./sessionAttribution.js";
import { FilterParameter, FilterType } from "../types.js";
// Options for customizing filter behavior
export interface FilterStatementOptions {
// Parameters that should use session-level subqueries (finds sessions containing matching events)
// Default: ["event_name", "channel"] - entry_page and exit_page are always session-level due to special aggregation
- // Channel is handled as a session acquisition field using the first non-empty channel in the session.
+ // Channel is handled as a session acquisition field using the session's first attributed channel
+ // (SESSION_CHANNEL_AGG), matching how session views derive their channel.
sessionLevelParams?: FilterParameter[];
// Field name mappings for CTEs that extract fields to different column names
@@ -186,26 +188,23 @@ export function getFilterStatement(
)`;
};
- const buildSessionFirstValueSubquery = (
- param: FilterParameter,
- alias: string,
+ const buildSessionChannelSubquery = (
filterType: FilterType,
values: (string | number)[],
wildcardPrefix: string
): string => {
- const whereClause = [siteIdFilter, timeFilter, `${param} IS NOT NULL`, `${param} <> ''`]
- .filter(Boolean)
- .join(" AND ");
- const condition = buildStringFilterCondition(alias, filterType, values, wildcardPrefix);
+ const whereClause = [siteIdFilter, timeFilter].filter(Boolean).join(" AND ");
+ const whereStatement = whereClause ? `WHERE ${whereClause}` : "";
+ const condition = buildStringFilterCondition("session_channel", filterType, values, wildcardPrefix);
return `session_id IN (
SELECT session_id
FROM (
SELECT
session_id,
- argMin(${param}, timestamp) AS ${alias}
+ ${SESSION_CHANNEL_AGG} AS session_channel
FROM events
- WHERE ${whereClause}
+ ${whereStatement}
GROUP BY session_id
)
WHERE ${condition}
@@ -221,10 +220,10 @@ export function getFilterStatement(
const isNullCheck = filter.type === "is_null" || filter.type === "is_not_null";
// Handle session-level filters (configurable via options).
- // Most parameters match sessions containing an event; channel uses the session's first value.
+ // Most parameters match sessions containing an event; channel uses the session's first attributed value.
if (sessionLevelParams.includes(filter.parameter)) {
if (filter.parameter === "channel") {
- return buildSessionFirstValueSubquery(filter.parameter, "session_channel", filter.type, filter.value, x);
+ return buildSessionChannelSubquery(filter.type, filter.value, x);
}
return buildSessionLevelSubquery(filter.parameter, filter.type, filter.value, x);
diff --git a/server/src/api/analytics/utils/sessionAttribution.ts b/server/src/api/analytics/utils/sessionAttribution.ts
new file mode 100644
index 000000000..79f918348
--- /dev/null
+++ b/server/src/api/analytics/utils/sessionAttribution.ts
@@ -0,0 +1,20 @@
+// Session-level attribution aggregations, shared by every query that derives a
+// session's (or user's) referrer/channel from its events. Keep these in sync with
+// the per-parameter reports in getMetric.ts: the referrers report counts a session
+// under its first non-empty referrer, so session views must derive referrer the
+// same way or the same session shows different sources in different views.
+
+// First non-empty referrer in the session; '' when every event is direct.
+// Plain argMin(referrer, timestamp) is wrong here: a session that starts direct
+// and later returns via an external link (30-minute session window) would show
+// as Direct while the referrers report attributes it to that link.
+export const SESSION_REFERRER_AGG = "argMinIf(referrer, timestamp, referrer != '')";
+
+// Channel values that carry no acquisition signal: mid-session pageviews are
+// 'Direct' (self-referrer cleared before getChannel) or 'Internal' (same-site
+// referrer that survived clearing, e.g. www variants). '' guards legacy rows.
+const UNATTRIBUTED_CHANNELS = "('Direct', 'Internal', '')";
+
+// First attributed channel in the session, falling back to the first event's
+// channel when no event in the session carries an acquisition signal.
+export const SESSION_CHANNEL_AGG = `if(countIf(channel NOT IN ${UNATTRIBUTED_CHANNELS}) > 0, argMinIf(channel, timestamp, channel NOT IN ${UNATTRIBUTED_CHANNELS}), argMin(channel, timestamp))`;