🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix(analytics): preserve sub-second event ordering (#1144)
Goldflag66e0cffa8cd1
diff --git a/server/src/api/analytics/funnels/getFunnel.test.ts b/server/src/api/analytics/funnels/getFunnel.test.tsnew file mode 100644index 000000000..aa195d849--- /dev/null+++ b/server/src/api/analytics/funnels/getFunnel.test.ts@@ -0,0 +1,19 @@+import { describe, expect, it, vi } from "vitest";++vi.mock("../../../db/clickhouse/clickhouse.js", () => ({+ clickhouse: { query: vi.fn() },+}));++import { buildFunnelQuery } from "./getFunnel.js";++describe("buildFunnelQuery", () => {+ it("orders funnel steps by the millisecond event timestamp", () => {+ const sql = buildFunnelQuery({ start_date: "", end_date: "", time_zone: "", filters: "" }, 1, [+ { type: "page", value: "/cart", name: "Cart" },+ { type: "page", value: "/checkout", name: "Checkout" },+ ]);++ expect(sql).toContain("timestamp_ms AS timestamp");+ expect(sql).toContain("sa.timestamp > s1.step_time");+ });+});diff --git a/server/src/api/analytics/funnels/getFunnel.ts b/server/src/api/analytics/funnels/getFunnel.tsindex c9156ea66..5809219e3 100644--- a/server/src/api/analytics/funnels/getFunnel.ts+++ b/server/src/api/analytics/funnels/getFunnel.ts@@ -34,7 +34,7 @@ export const buildFunnelQuery = (query: FilterParams<{}>, siteId: number, steps:SessionActions AS (SELECTsession_id,- timestamp,+ timestamp_ms AS timestamp,pathname,event_name,type,diff --git a/server/src/api/analytics/funnels/getFunnelStepSessions.ts b/server/src/api/analytics/funnels/getFunnelStepSessions.tsindex 2612d1117..2adae8fdc 100644--- a/server/src/api/analytics/funnels/getFunnelStepSessions.ts+++ b/server/src/api/analytics/funnels/getFunnelStepSessions.ts@@ -144,8 +144,8 @@ export const buildFunnelStepSessionsQuery = (MAX(e.timestamp) AS session_end,MIN(e.timestamp) AS session_start,dateDiff('second', MIN(e.timestamp), MAX(e.timestamp)) AS session_duration,- argMinIf(e.pathname, e.timestamp, e.type = 'pageview') AS entry_page,- argMaxIf(e.pathname, e.timestamp, e.type = 'pageview') AS exit_page,+ argMinIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS entry_page,+ argMaxIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS exit_page,countIf(e.type = 'pageview') AS pageviews,countIf(e.type = 'custom_event') AS events,countIf(e.type = 'error') AS errors,diff --git a/server/src/api/analytics/getMetric.test.ts b/server/src/api/analytics/getMetric.test.tsindex ee675a0f5..25e499d89 100644--- a/server/src/api/analytics/getMetric.test.ts+++ b/server/src/api/analytics/getMetric.test.ts@@ -89,13 +89,14 @@ describe("buildMetricQuery", () => {describe("entry_page / exit_page", () => {it("orders sessions ascending for entry pages", () => {const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);- expect(sql).toContain("ORDER BY timestamp ASC) as row_num");+ expect(sql).toContain("timestamp_ms AS timestamp");+ expect(sql).toContain("ORDER BY timestamp_ms ASC) as row_num");expect(sql).toContain("WHERE row_num = 1");});it("orders sessions descending for exit pages", () => {const sql = buildMetricQuery(baseQuery({ parameter: "exit_page" }), SITE_ID);- expect(sql).toContain("ORDER BY timestamp DESC) as row_num");+ expect(sql).toContain("ORDER BY timestamp_ms DESC) as row_num");});it("caps time-on-page at 30 minutes", () => {diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.tsindex 9ae1e3897..c91f17845 100644--- a/server/src/api/analytics/getMetric.ts+++ b/server/src/api/analytics/getMetric.ts@@ -219,10 +219,10 @@ export const buildMetricQuery = (session_id,pathname,hostname,- timestamp,+ timestamp_ms AS timestamp,pageviews_in_session,- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp,- row_number() OVER (PARTITION BY session_id ORDER BY timestamp ${orderDirection}) as row_num+ leadInFrame(timestamp_ms) OVER (PARTITION BY session_id ORDER BY timestamp_ms ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp,+ row_number() OVER (PARTITION BY session_id ORDER BY timestamp_ms ${orderDirection}) as row_numFROM RelevantEvents),PageDurations AS (diff --git a/server/src/api/analytics/goals/getGoalSessions.ts b/server/src/api/analytics/goals/getGoalSessions.tsindex 36fa11baf..43cd8c2f2 100644--- a/server/src/api/analytics/goals/getGoalSessions.ts+++ b/server/src/api/analytics/goals/getGoalSessions.ts@@ -69,8 +69,8 @@ export const buildGoalSessionsQuery = (MAX(e.timestamp) AS session_end,MIN(e.timestamp) AS session_start,dateDiff('second', MIN(e.timestamp), MAX(e.timestamp)) AS session_duration,- argMinIf(e.pathname, e.timestamp, e.type = 'pageview') AS entry_page,- argMaxIf(e.pathname, e.timestamp, e.type = 'pageview') AS exit_page,+ argMinIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS entry_page,+ argMaxIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS exit_page,countIf(e.type = 'pageview') AS pageviews,countIf(e.type = 'custom_event') AS events,countIf(e.type = 'error') AS errors,diff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.tsindex 71946b3d0..0f22da42c 100644--- a/server/src/api/analytics/sessions/getSession.ts+++ b/server/src/api/analytics/sessions/getSession.ts@@ -95,8 +95,8 @@ SELECTdateDiff('second', min(timestamp), max(timestamp)) as session_duration,countIf(type = 'pageview') as pageviews,count() as events,- argMinIf(pathname, timestamp, type = 'pageview') as entry_page,- argMaxIf(pathname, timestamp, type = 'pageview') as exit_page,+ argMinIf(pathname, timestamp_ms, type = 'pageview') as entry_page,+ argMaxIf(pathname, timestamp_ms, type = 'pageview') as exit_page,any(ip) AS ipFROM eventsWHERE@@ -122,7 +122,7 @@ WHERE// 3. Query to get paginated pageviewsconst eventsQuery = `SELECT- timestamp,+ timestamp_ms AS timestamp,pathname,hostname,querystring,@@ -137,7 +137,7 @@ WHEREAND session_id = {sessionId:String}AND type != 'performance'${timeFilterWithConnector}-ORDER BY timestamp ASC+ORDER BY timestamp_ms ASCLIMIT {limit:Int32}OFFSET {offset:Int32}`;diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.tsindex 41434d9c9..b2aee217a 100644--- a/server/src/api/analytics/sessions/getSessions.ts+++ b/server/src/api/analytics/sessions/getSessions.ts@@ -122,8 +122,8 @@ export const buildSessionsQuery = (query: GetSessionsRequest["Querystring"], sitMAX(timestamp) AS session_end,MIN(timestamp) AS session_start,dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,- argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,- argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,+ argMinIf(pathname, timestamp_ms, type = 'pageview') AS entry_page,+ argMaxIf(pathname, timestamp_ms, type = 'pageview') AS exit_page,countIf(type = 'pageview') AS pageviews,countIf(type = 'custom_event') AS events,countIf(type = 'error') AS errors,diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.tsindex a82db847b..d5853065b 100644--- a/server/src/api/analytics/users/getUserInfo.ts+++ b/server/src/api/analytics/users/getUserInfo.ts@@ -133,8 +133,8 @@ export const buildUserInfoQueries = (query: FilterParams, siteId: number) => {MAX(timestamp) AS session_end,MIN(timestamp) AS session_start,dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,- argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,- argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,+ argMinIf(pathname, timestamp_ms, type = 'pageview') AS entry_page,+ argMaxIf(pathname, timestamp_ms, type = 'pageview') AS exit_page,countIf(type = 'pageview') AS pageviews,countIf(type = 'custom_event') AS events,argMax(ip, timestamp) AS ipdiff --git a/server/src/api/analytics/utils/eventSchema.ts b/server/src/api/analytics/utils/eventSchema.tsindex 443fc4b40..14d1eff60 100644--- a/server/src/api/analytics/utils/eventSchema.ts+++ b/server/src/api/analytics/utils/eventSchema.ts@@ -4,6 +4,7 @@ export const EVENT_SCHEMA = `scoped_events columns:- site_id UInt16: Numeric Rybbit site id. scoped_events is already limited to sites the user can access.- timestamp DateTime: Event ingest time in ClickHouse. Use ClickHouse date functions such as toStartOfDay(timestamp).+- timestamp_ms DateTime64(3): Millisecond event ingest time. Use for event ordering and entry/exit attribution.- session_id String: Anonymous visit/session id. Use countDistinct(session_id) for sessions.- user_id String: Anonymous device/user fingerprint id. This is not necessarily a logged-in app user id.- identified_user_id String: Custom user id set via identify(); empty string when the visitor was not identified.diff --git a/server/src/api/analytics/utils/getFilterStatement.test.ts b/server/src/api/analytics/utils/getFilterStatement.test.tsindex efa8766bc..f60fe0feb 100644--- a/server/src/api/analytics/utils/getFilterStatement.test.ts+++ b/server/src/api/analytics/utils/getFilterStatement.test.ts@@ -49,13 +49,13 @@ describe("getSqlParam", () => {it("should handle entry_page", () => {expect(getSqlParam("entry_page")).toBe(- "(SELECT argMinIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)"+ "(SELECT argMinIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)");});it("should handle exit_page", () => {expect(getSqlParam("exit_page")).toBe(- "(SELECT argMaxIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)"+ "(SELECT argMaxIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)");});@@ -326,7 +326,7 @@ describe("getFilterStatement", () => {const filters = JSON.stringify([{ parameter: "entry_page", type: "equals", value: ["/home"] }]);const result = getFilterStatement(filters);expect(result).toContain("session_id IN");- expect(result).toContain("argMin(pathname, timestamp) AS entry_pathname");+ expect(result).toContain("argMin(pathname, timestamp_ms) AS entry_pathname");expect(result).toContain("type = 'pageview'");expect(result).toContain("entry_pathname = '/home'");});@@ -351,7 +351,7 @@ describe("getFilterStatement", () => {const filters = JSON.stringify([{ parameter: "exit_page", type: "equals", value: ["/checkout"] }]);const result = getFilterStatement(filters);expect(result).toContain("session_id IN");- expect(result).toContain("argMax(pathname, timestamp) AS exit_pathname");+ expect(result).toContain("argMax(pathname, timestamp_ms) AS exit_pathname");expect(result).toContain("type = 'pageview'");expect(result).toContain("exit_pathname = '/checkout'");});diff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.tsindex ccadd9592..b301e27b1 100644--- a/server/src/api/analytics/utils/getFilterStatement.ts+++ b/server/src/api/analytics/utils/getFilterStatement.ts@@ -150,10 +150,10 @@ export const getSqlParam = (parameter: FilterParameter) => {return "domainWithoutWWW(referrer)";}if (parameter === "entry_page") {- return "(SELECT argMinIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)";+ return "(SELECT argMinIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)";}if (parameter === "exit_page") {- return "(SELECT argMaxIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)";+ return "(SELECT argMaxIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)";}if (parameter === "dimensions") {return "concat(toString(screen_width), 'x', toString(screen_height))";@@ -279,7 +279,7 @@ export function getFilterStatement(FROM (SELECTsession_id,- argMin(pathname, timestamp) AS entry_pathname+ argMin(pathname, timestamp_ms) AS entry_pathnameFROM events${whereStatement}GROUP BY session_id@@ -298,7 +298,7 @@ export function getFilterStatement(FROM (SELECTsession_id,- argMax(pathname, timestamp) AS exit_pathname+ argMax(pathname, timestamp_ms) AS exit_pathnameFROM events${whereStatement}GROUP BY session_iddiff --git a/server/src/api/analytics/utils/timeWindow.test.ts b/server/src/api/analytics/utils/timeWindow.test.tsindex 32b902c13..e8372ef76 100644--- a/server/src/api/analytics/utils/timeWindow.test.ts+++ b/server/src/api/analytics/utils/timeWindow.test.ts@@ -32,10 +32,10 @@ describe("where()", () => {toStartOfDay(toDateTime('2024-01-01', 'America/New_York')),'UTC')- AND timestamp < if(+ AND if(toDate('2024-01-31') = toDate(now(), 'America/New_York'),- toTimeZone(now(), 'UTC'),- toTimeZone(+ timestamp <= toTimeZone(now64(3), 'UTC'),+ timestamp < toTimeZone(toStartOfDay(toDateTime('2024-01-31', 'America/New_York')) + INTERVAL 1 DAY,'UTC')@@ -55,7 +55,10 @@ describe("where()", () => {const result = getTime({ start_date: "2024-06-15", end_date: "2024-06-15", time_zone: "UTC" });expect(normalize(result)).toContain("toDate('2024-06-15') = toDate(now(), 'UTC')");- expect(normalize(result)).toContain("toTimeZone(now(), 'UTC')");+ expect(normalize(result)).toContain("timestamp <= toTimeZone(now64(3), 'UTC')");+ expect(normalize(result)).toContain(+ "timestamp < toTimeZone( toStartOfDay(toDateTime('2024-06-15', 'UTC')) + INTERVAL 1 DAY, 'UTC' )"+ );});it("should take precedence over datetime range and past minutes", () => {@@ -258,9 +261,9 @@ describe("fill()", () => {});it("should not fill past an upper bound already on a bucket boundary", () => {- expect(- fillOf({ start_datetime: "2024-01-01 00:00:00", end_datetime: "2024-01-01 04:00:00" }, "hour")- ).toContain("TO if(");+ expect(fillOf({ start_datetime: "2024-01-01 00:00:00", end_datetime: "2024-01-01 04:00:00" }, "hour")).toContain(+ "TO if("+ );});});diff --git a/server/src/api/analytics/utils/timeWindow.ts b/server/src/api/analytics/utils/timeWindow.tsindex 1de99df49..15c06b8cc 100644--- a/server/src/api/analytics/utils/timeWindow.ts+++ b/server/src/api/analytics/utils/timeWindow.ts@@ -290,10 +290,10 @@ function whereClause(window: ResolvedWindow, column: string): string {toStartOfDay(toDateTime(${SqlString.escape(startDate)}, ${tz})),'UTC')- AND ${column} < if(+ AND if(toDate(${SqlString.escape(endDate)}) = toDate(now(), ${tz}),- toTimeZone(now(), 'UTC'),- toTimeZone(+ ${column} <= toTimeZone(now64(3), 'UTC'),+ ${column} < toTimeZone(toStartOfDay(toDateTime(${SqlString.escape(endDate)}, ${tz})) + INTERVAL 1 DAY,'UTC')diff --git a/server/src/db/clickhouse/clickhouse.test.ts b/server/src/db/clickhouse/clickhouse.test.tsindex 23b04c5f3..b8b4b248c 100644--- a/server/src/db/clickhouse/clickhouse.test.ts+++ b/server/src/db/clickhouse/clickhouse.test.ts@@ -72,6 +72,17 @@ describe("session hourly materialized view initialization", () => {expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));});+ it("creates and heals the millisecond event-ordering column", async () => {+ await initializeClickhouse();++ const queries = executedQueries();+ const createEvents = queries.find(query => /CREATE TABLE IF NOT EXISTS events\b/.test(query));+ const alterEvents = queries.find(query => /ALTER TABLE events\b/.test(query));++ expect(createEvents).toContain("timestamp_ms DateTime64(3)");+ expect(alterEvents).toContain("timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3)");+ });+it("replaces the old five-minute raw-events definition", async () => {state.existingSessionHourlyView = `CREATE MATERIALIZED VIEW analytics.session_hourly_mvdiff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.tsindex c355608db..35c7401c4 100644--- a/server/src/db/clickhouse/schema/core.ts+++ b/server/src/db/clickhouse/schema/core.ts@@ -6,6 +6,10 @@ type ColumnDefinition = {};const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [+ // `timestamp` is part of the MergeTree key and ClickHouse forbids widening a+ // key column in place. Keep it for partition pruning; use this additive+ // column wherever event order matters. Old rows read as whole-second values.+ { name: "timestamp_ms", definition: "timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3) AFTER timestamp" },{ name: "lcp", definition: "lcp Nullable(Float64)" },{ name: "cls", definition: "cls Nullable(Float64)" },{ name: "inp", definition: "inp Nullable(Float64)" },@@ -98,6 +102,7 @@ export async function initializeCoreTables() {CREATE TABLE IF NOT EXISTS events (site_id UInt16,timestamp DateTime,+ timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3),session_id String,user_id String,hostname String,diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.tsindex 6d0595bbf..39046d33e 100644--- a/server/src/services/tracker/pageviewQueue.test.ts+++ b/server/src/services/tracker/pageviewQueue.test.ts@@ -95,4 +95,13 @@ describe("pageviewQueue ASN enrichment", () => {const row = mocks.insert.mock.calls[0][0].values[0];expect(row.is_datacenter_asn).toBe(0);});++ it("preserves the request's millisecond timestamp for event ordering", async () => {+ await pageviewQueue.add(makePayload({ timestamp: "2026-08-28T12:34:56.789Z" }));+ await vi.advanceTimersByTimeAsync(1000);++ const row = mocks.insert.mock.calls[0][0].values[0];+ expect(row.timestamp).toBe("2026-08-28 12:34:56");+ expect(row.timestamp_ms).toBe("2026-08-28 12:34:56.789");+ });});diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.tsindex af3466ee0..7b3be3a9c 100644--- a/server/src/services/tracker/pageviewQueue.ts+++ b/server/src/services/tracker/pageviewQueue.ts@@ -79,10 +79,15 @@ class PageviewQueue {// Get all URL parameters for the url_parameters mapconst allUrlParams = getAllUrlParams(pv.querystring || "");+ const receivedAt = DateTime.fromISO(pv.timestamp).toUTC();return {site_id: pv.site_id,- timestamp: DateTime.fromISO(pv.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"),+ // Keep the original DateTime column for the MergeTree key and+ // whole-second time filters; timestamp_ms preserves arrival order+ // for funnels and entry/exit attribution.+ timestamp: receivedAt.toFormat("yyyy-MM-dd HH:mm:ss"),+ timestamp_ms: receivedAt.toFormat("yyyy-MM-dd HH:mm:ss.SSS"),session_id: pv.sessionId,user_id: pv.userId, // Always the device fingerprintidentified_user_id: pv.identifiedUserId || "", // Custom user ID when identified