Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

fix(analytics): preserve sub-second event ordering (#1144)

Goldflag66e0cffa8cd1
diff --git a/server/src/api/analytics/funnels/getFunnel.test.ts b/server/src/api/analytics/funnels/getFunnel.test.ts
new file mode 100644
index 000000000..aa195d849
--- /dev/null
+++ b/server/src/api/analytics/funnels/getFunnel.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../../../db/clickhouse/clickhouse.js", () => ({
+ clickhouse: { query: vi.fn() },
+}));
+
+import { buildFunnelQuery } from "./getFunnel.js";
+
+describe("buildFunnelQuery", () => {
+ it("orders funnel steps by the millisecond event timestamp", () => {
+ const sql = buildFunnelQuery({ start_date: "", end_date: "", time_zone: "", filters: "" }, 1, [
+ { type: "page", value: "/cart", name: "Cart" },
+ { type: "page", value: "/checkout", name: "Checkout" },
+ ]);
+
+ expect(sql).toContain("timestamp_ms AS timestamp");
+ expect(sql).toContain("sa.timestamp > s1.step_time");
+ });
+});
diff --git a/server/src/api/analytics/funnels/getFunnel.ts b/server/src/api/analytics/funnels/getFunnel.ts
index c9156ea66..5809219e3 100644
--- a/server/src/api/analytics/funnels/getFunnel.ts
+++ b/server/src/api/analytics/funnels/getFunnel.ts
@@ -34,7 +34,7 @@ export const buildFunnelQuery = (query: FilterParams<{}>, siteId: number, steps:
SessionActions AS (
SELECT
session_id,
- timestamp,
+ timestamp_ms AS timestamp,
pathname,
event_name,
type,
diff --git a/server/src/api/analytics/funnels/getFunnelStepSessions.ts b/server/src/api/analytics/funnels/getFunnelStepSessions.ts
index 2612d1117..2adae8fdc 100644
--- a/server/src/api/analytics/funnels/getFunnelStepSessions.ts
+++ b/server/src/api/analytics/funnels/getFunnelStepSessions.ts
@@ -144,8 +144,8 @@ export const buildFunnelStepSessionsQuery = (
MAX(e.timestamp) AS session_end,
MIN(e.timestamp) AS session_start,
dateDiff('second', MIN(e.timestamp), MAX(e.timestamp)) AS session_duration,
- argMinIf(e.pathname, e.timestamp, e.type = 'pageview') AS entry_page,
- argMaxIf(e.pathname, e.timestamp, e.type = 'pageview') AS exit_page,
+ argMinIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS entry_page,
+ argMaxIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS exit_page,
countIf(e.type = 'pageview') AS pageviews,
countIf(e.type = 'custom_event') AS events,
countIf(e.type = 'error') AS errors,
diff --git a/server/src/api/analytics/getMetric.test.ts b/server/src/api/analytics/getMetric.test.ts
index ee675a0f5..25e499d89 100644
--- a/server/src/api/analytics/getMetric.test.ts
+++ b/server/src/api/analytics/getMetric.test.ts
@@ -89,13 +89,14 @@ describe("buildMetricQuery", () => {
describe("entry_page / exit_page", () => {
it("orders sessions ascending for entry pages", () => {
const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);
- expect(sql).toContain("ORDER BY timestamp ASC) as row_num");
+ expect(sql).toContain("timestamp_ms AS timestamp");
+ expect(sql).toContain("ORDER BY timestamp_ms ASC) as row_num");
expect(sql).toContain("WHERE row_num = 1");
});
it("orders sessions descending for exit pages", () => {
const sql = buildMetricQuery(baseQuery({ parameter: "exit_page" }), SITE_ID);
- expect(sql).toContain("ORDER BY timestamp DESC) as row_num");
+ expect(sql).toContain("ORDER BY timestamp_ms DESC) as row_num");
});
it("caps time-on-page at 30 minutes", () => {
diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.ts
index 9ae1e3897..c91f17845 100644
--- a/server/src/api/analytics/getMetric.ts
+++ b/server/src/api/analytics/getMetric.ts
@@ -219,10 +219,10 @@ export const buildMetricQuery = (
session_id,
pathname,
hostname,
- timestamp,
+ timestamp_ms AS timestamp,
pageviews_in_session,
- leadInFrame(timestamp) OVER (PARTITION BY session_id ORDER BY timestamp ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp,
- row_number() OVER (PARTITION BY session_id ORDER BY timestamp ${orderDirection}) as row_num
+ leadInFrame(timestamp_ms) OVER (PARTITION BY session_id ORDER BY timestamp_ms ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING) as next_timestamp,
+ row_number() OVER (PARTITION BY session_id ORDER BY timestamp_ms ${orderDirection}) as row_num
FROM RelevantEvents
),
PageDurations AS (
diff --git a/server/src/api/analytics/goals/getGoalSessions.ts b/server/src/api/analytics/goals/getGoalSessions.ts
index 36fa11baf..43cd8c2f2 100644
--- a/server/src/api/analytics/goals/getGoalSessions.ts
+++ b/server/src/api/analytics/goals/getGoalSessions.ts
@@ -69,8 +69,8 @@ export const buildGoalSessionsQuery = (
MAX(e.timestamp) AS session_end,
MIN(e.timestamp) AS session_start,
dateDiff('second', MIN(e.timestamp), MAX(e.timestamp)) AS session_duration,
- argMinIf(e.pathname, e.timestamp, e.type = 'pageview') AS entry_page,
- argMaxIf(e.pathname, e.timestamp, e.type = 'pageview') AS exit_page,
+ argMinIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS entry_page,
+ argMaxIf(e.pathname, e.timestamp_ms, e.type = 'pageview') AS exit_page,
countIf(e.type = 'pageview') AS pageviews,
countIf(e.type = 'custom_event') AS events,
countIf(e.type = 'error') AS errors,
diff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.ts
index 71946b3d0..0f22da42c 100644
--- a/server/src/api/analytics/sessions/getSession.ts
+++ b/server/src/api/analytics/sessions/getSession.ts
@@ -95,8 +95,8 @@ SELECT
dateDiff('second', min(timestamp), max(timestamp)) as session_duration,
countIf(type = 'pageview') as pageviews,
count() as events,
- argMinIf(pathname, timestamp, type = 'pageview') as entry_page,
- argMaxIf(pathname, timestamp, type = 'pageview') as exit_page,
+ argMinIf(pathname, timestamp_ms, type = 'pageview') as entry_page,
+ argMaxIf(pathname, timestamp_ms, type = 'pageview') as exit_page,
any(ip) AS ip
FROM events
WHERE
@@ -122,7 +122,7 @@ WHERE
// 3. Query to get paginated pageviews
const eventsQuery = `
SELECT
- timestamp,
+ timestamp_ms AS timestamp,
pathname,
hostname,
querystring,
@@ -137,7 +137,7 @@ WHERE
AND session_id = {sessionId:String}
AND type != 'performance'
${timeFilterWithConnector}
-ORDER BY timestamp ASC
+ORDER BY timestamp_ms ASC
LIMIT {limit:Int32}
OFFSET {offset:Int32}
`;
diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.ts
index 41434d9c9..b2aee217a 100644
--- a/server/src/api/analytics/sessions/getSessions.ts
+++ b/server/src/api/analytics/sessions/getSessions.ts
@@ -122,8 +122,8 @@ export const buildSessionsQuery = (query: GetSessionsRequest["Querystring"], sit
MAX(timestamp) AS session_end,
MIN(timestamp) AS session_start,
dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,
- argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,
- argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,
+ argMinIf(pathname, timestamp_ms, type = 'pageview') AS entry_page,
+ argMaxIf(pathname, timestamp_ms, type = 'pageview') AS exit_page,
countIf(type = 'pageview') AS pageviews,
countIf(type = 'custom_event') AS events,
countIf(type = 'error') AS errors,
diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.ts
index a82db847b..d5853065b 100644
--- a/server/src/api/analytics/users/getUserInfo.ts
+++ b/server/src/api/analytics/users/getUserInfo.ts
@@ -133,8 +133,8 @@ export const buildUserInfoQueries = (query: FilterParams, siteId: number) => {
MAX(timestamp) AS session_end,
MIN(timestamp) AS session_start,
dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,
- argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,
- argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,
+ argMinIf(pathname, timestamp_ms, type = 'pageview') AS entry_page,
+ argMaxIf(pathname, timestamp_ms, type = 'pageview') AS exit_page,
countIf(type = 'pageview') AS pageviews,
countIf(type = 'custom_event') AS events,
argMax(ip, timestamp) AS ip
diff --git a/server/src/api/analytics/utils/eventSchema.ts b/server/src/api/analytics/utils/eventSchema.ts
index 443fc4b40..14d1eff60 100644
--- a/server/src/api/analytics/utils/eventSchema.ts
+++ b/server/src/api/analytics/utils/eventSchema.ts
@@ -4,6 +4,7 @@ export const EVENT_SCHEMA = `
scoped_events columns:
- site_id UInt16: Numeric Rybbit site id. scoped_events is already limited to sites the user can access.
- timestamp DateTime: Event ingest time in ClickHouse. Use ClickHouse date functions such as toStartOfDay(timestamp).
+- timestamp_ms DateTime64(3): Millisecond event ingest time. Use for event ordering and entry/exit attribution.
- session_id String: Anonymous visit/session id. Use countDistinct(session_id) for sessions.
- user_id String: Anonymous device/user fingerprint id. This is not necessarily a logged-in app user id.
- identified_user_id String: Custom user id set via identify(); empty string when the visitor was not identified.
diff --git a/server/src/api/analytics/utils/getFilterStatement.test.ts b/server/src/api/analytics/utils/getFilterStatement.test.ts
index efa8766bc..f60fe0feb 100644
--- a/server/src/api/analytics/utils/getFilterStatement.test.ts
+++ b/server/src/api/analytics/utils/getFilterStatement.test.ts
@@ -49,13 +49,13 @@ describe("getSqlParam", () => {
it("should handle entry_page", () => {
expect(getSqlParam("entry_page")).toBe(
- "(SELECT argMinIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)"
+ "(SELECT argMinIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)"
);
});
it("should handle exit_page", () => {
expect(getSqlParam("exit_page")).toBe(
- "(SELECT argMaxIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)"
+ "(SELECT argMaxIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)"
);
});
@@ -326,7 +326,7 @@ describe("getFilterStatement", () => {
const filters = JSON.stringify([{ parameter: "entry_page", type: "equals", value: ["/home"] }]);
const result = getFilterStatement(filters);
expect(result).toContain("session_id IN");
- expect(result).toContain("argMin(pathname, timestamp) AS entry_pathname");
+ expect(result).toContain("argMin(pathname, timestamp_ms) AS entry_pathname");
expect(result).toContain("type = 'pageview'");
expect(result).toContain("entry_pathname = '/home'");
});
@@ -351,7 +351,7 @@ describe("getFilterStatement", () => {
const filters = JSON.stringify([{ parameter: "exit_page", type: "equals", value: ["/checkout"] }]);
const result = getFilterStatement(filters);
expect(result).toContain("session_id IN");
- expect(result).toContain("argMax(pathname, timestamp) AS exit_pathname");
+ expect(result).toContain("argMax(pathname, timestamp_ms) AS exit_pathname");
expect(result).toContain("type = 'pageview'");
expect(result).toContain("exit_pathname = '/checkout'");
});
diff --git a/server/src/api/analytics/utils/getFilterStatement.ts b/server/src/api/analytics/utils/getFilterStatement.ts
index ccadd9592..b301e27b1 100644
--- a/server/src/api/analytics/utils/getFilterStatement.ts
+++ b/server/src/api/analytics/utils/getFilterStatement.ts
@@ -150,10 +150,10 @@ export const getSqlParam = (parameter: FilterParameter) => {
return "domainWithoutWWW(referrer)";
}
if (parameter === "entry_page") {
- return "(SELECT argMinIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)";
+ return "(SELECT argMinIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)";
}
if (parameter === "exit_page") {
- return "(SELECT argMaxIf(pathname, timestamp, type = 'pageview') FROM events WHERE session_id = events.session_id)";
+ return "(SELECT argMaxIf(pathname, timestamp_ms, type = 'pageview') FROM events WHERE session_id = events.session_id)";
}
if (parameter === "dimensions") {
return "concat(toString(screen_width), 'x', toString(screen_height))";
@@ -279,7 +279,7 @@ export function getFilterStatement(
FROM (
SELECT
session_id,
- argMin(pathname, timestamp) AS entry_pathname
+ argMin(pathname, timestamp_ms) AS entry_pathname
FROM events
${whereStatement}
GROUP BY session_id
@@ -298,7 +298,7 @@ export function getFilterStatement(
FROM (
SELECT
session_id,
- argMax(pathname, timestamp) AS exit_pathname
+ argMax(pathname, timestamp_ms) AS exit_pathname
FROM events
${whereStatement}
GROUP BY session_id
diff --git a/server/src/api/analytics/utils/timeWindow.test.ts b/server/src/api/analytics/utils/timeWindow.test.ts
index 32b902c13..e8372ef76 100644
--- a/server/src/api/analytics/utils/timeWindow.test.ts
+++ b/server/src/api/analytics/utils/timeWindow.test.ts
@@ -32,10 +32,10 @@ describe("where()", () => {
toStartOfDay(toDateTime('2024-01-01', 'America/New_York')),
'UTC'
)
- AND timestamp < if(
+ AND if(
toDate('2024-01-31') = toDate(now(), 'America/New_York'),
- toTimeZone(now(), 'UTC'),
- toTimeZone(
+ timestamp <= toTimeZone(now64(3), 'UTC'),
+ timestamp < toTimeZone(
toStartOfDay(toDateTime('2024-01-31', 'America/New_York')) + INTERVAL 1 DAY,
'UTC'
)
@@ -55,7 +55,10 @@ describe("where()", () => {
const result = getTime({ start_date: "2024-06-15", end_date: "2024-06-15", time_zone: "UTC" });
expect(normalize(result)).toContain("toDate('2024-06-15') = toDate(now(), 'UTC')");
- expect(normalize(result)).toContain("toTimeZone(now(), 'UTC')");
+ expect(normalize(result)).toContain("timestamp <= toTimeZone(now64(3), 'UTC')");
+ expect(normalize(result)).toContain(
+ "timestamp < toTimeZone( toStartOfDay(toDateTime('2024-06-15', 'UTC')) + INTERVAL 1 DAY, 'UTC' )"
+ );
});
it("should take precedence over datetime range and past minutes", () => {
@@ -258,9 +261,9 @@ describe("fill()", () => {
});
it("should not fill past an upper bound already on a bucket boundary", () => {
- expect(
- fillOf({ start_datetime: "2024-01-01 00:00:00", end_datetime: "2024-01-01 04:00:00" }, "hour")
- ).toContain("TO if(");
+ expect(fillOf({ start_datetime: "2024-01-01 00:00:00", end_datetime: "2024-01-01 04:00:00" }, "hour")).toContain(
+ "TO if("
+ );
});
});
diff --git a/server/src/api/analytics/utils/timeWindow.ts b/server/src/api/analytics/utils/timeWindow.ts
index 1de99df49..15c06b8cc 100644
--- a/server/src/api/analytics/utils/timeWindow.ts
+++ b/server/src/api/analytics/utils/timeWindow.ts
@@ -290,10 +290,10 @@ function whereClause(window: ResolvedWindow, column: string): string {
toStartOfDay(toDateTime(${SqlString.escape(startDate)}, ${tz})),
'UTC'
)
- AND ${column} < if(
+ AND if(
toDate(${SqlString.escape(endDate)}) = toDate(now(), ${tz}),
- toTimeZone(now(), 'UTC'),
- toTimeZone(
+ ${column} <= toTimeZone(now64(3), 'UTC'),
+ ${column} < toTimeZone(
toStartOfDay(toDateTime(${SqlString.escape(endDate)}, ${tz})) + INTERVAL 1 DAY,
'UTC'
)
diff --git a/server/src/db/clickhouse/clickhouse.test.ts b/server/src/db/clickhouse/clickhouse.test.ts
index 23b04c5f3..b8b4b248c 100644
--- a/server/src/db/clickhouse/clickhouse.test.ts
+++ b/server/src/db/clickhouse/clickhouse.test.ts
@@ -72,6 +72,17 @@ describe("session hourly materialized view initialization", () => {
expect(executedQueries()).not.toContainEqual(expect.stringMatching(/DROP VIEW/));
});
+ it("creates and heals the millisecond event-ordering column", async () => {
+ await initializeClickhouse();
+
+ const queries = executedQueries();
+ const createEvents = queries.find(query => /CREATE TABLE IF NOT EXISTS events\b/.test(query));
+ const alterEvents = queries.find(query => /ALTER TABLE events\b/.test(query));
+
+ expect(createEvents).toContain("timestamp_ms DateTime64(3)");
+ expect(alterEvents).toContain("timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3)");
+ });
+
it("replaces the old five-minute raw-events definition", async () => {
state.existingSessionHourlyView = `
CREATE MATERIALIZED VIEW analytics.session_hourly_mv
diff --git a/server/src/db/clickhouse/schema/core.ts b/server/src/db/clickhouse/schema/core.ts
index c355608db..35c7401c4 100644
--- a/server/src/db/clickhouse/schema/core.ts
+++ b/server/src/db/clickhouse/schema/core.ts
@@ -6,6 +6,10 @@ type ColumnDefinition = {
};
const EVENTS_COLUMNS_TO_ENSURE: ColumnDefinition[] = [
+ // `timestamp` is part of the MergeTree key and ClickHouse forbids widening a
+ // key column in place. Keep it for partition pruning; use this additive
+ // column wherever event order matters. Old rows read as whole-second values.
+ { name: "timestamp_ms", definition: "timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3) AFTER timestamp" },
{ name: "lcp", definition: "lcp Nullable(Float64)" },
{ name: "cls", definition: "cls Nullable(Float64)" },
{ name: "inp", definition: "inp Nullable(Float64)" },
@@ -98,6 +102,7 @@ export async function initializeCoreTables() {
CREATE TABLE IF NOT EXISTS events (
site_id UInt16,
timestamp DateTime,
+ timestamp_ms DateTime64(3) DEFAULT toDateTime64(timestamp, 3),
session_id String,
user_id String,
hostname String,
diff --git a/server/src/services/tracker/pageviewQueue.test.ts b/server/src/services/tracker/pageviewQueue.test.ts
index 6d0595bbf..39046d33e 100644
--- a/server/src/services/tracker/pageviewQueue.test.ts
+++ b/server/src/services/tracker/pageviewQueue.test.ts
@@ -95,4 +95,13 @@ describe("pageviewQueue ASN enrichment", () => {
const row = mocks.insert.mock.calls[0][0].values[0];
expect(row.is_datacenter_asn).toBe(0);
});
+
+ it("preserves the request's millisecond timestamp for event ordering", async () => {
+ await pageviewQueue.add(makePayload({ timestamp: "2026-08-28T12:34:56.789Z" }));
+ await vi.advanceTimersByTimeAsync(1000);
+
+ const row = mocks.insert.mock.calls[0][0].values[0];
+ expect(row.timestamp).toBe("2026-08-28 12:34:56");
+ expect(row.timestamp_ms).toBe("2026-08-28 12:34:56.789");
+ });
});
diff --git a/server/src/services/tracker/pageviewQueue.ts b/server/src/services/tracker/pageviewQueue.ts
index af3466ee0..7b3be3a9c 100644
--- a/server/src/services/tracker/pageviewQueue.ts
+++ b/server/src/services/tracker/pageviewQueue.ts
@@ -79,10 +79,15 @@ class PageviewQueue {
// Get all URL parameters for the url_parameters map
const allUrlParams = getAllUrlParams(pv.querystring || "");
+ const receivedAt = DateTime.fromISO(pv.timestamp).toUTC();
return {
site_id: pv.site_id,
- timestamp: DateTime.fromISO(pv.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"),
+ // Keep the original DateTime column for the MergeTree key and
+ // whole-second time filters; timestamp_ms preserves arrival order
+ // for funnels and entry/exit attribution.
+ timestamp: receivedAt.toFormat("yyyy-MM-dd HH:mm:ss"),
+ timestamp_ms: receivedAt.toFormat("yyyy-MM-dd HH:mm:ss.SSS"),
session_id: pv.sessionId,
user_id: pv.userId, // Always the device fingerprint
identified_user_id: pv.identifiedUserId || "", // Custom user ID when identified