🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix(analytics): read exact events for short rolling windows
Bill Yang7686128322bb
diff --git a/server/src/api/analytics/getSiteCards.test.ts b/server/src/api/analytics/getSiteCards.test.tsindex 935bbf6c5..1390296f5 100644--- a/server/src/api/analytics/getSiteCards.test.ts+++ b/server/src/api/analytics/getSiteCards.test.ts@@ -123,6 +123,32 @@ describe.each(["site-cards", "site-cards-lite"])("batched %s", endpoint => {expect(mocks.query.mock.calls[0][0].query).not.toContain("2026-09-18");});+ it.each([5, 30, 60, 120])("reads exact events for a %i-minute rolling window", async minutes => {+ const response = await request(+ {+ siteIds: [1, 2],+ comparison: { past_minutes_start: minutes * 2, past_minutes_end: minutes },+ },+ `/organizations/org-1/${endpoint}?past_minutes_start=${minutes}&past_minutes_end=0&bucket=minute`+ );+ expect(response.statusCode).toBe(200);+ expect(mocks.query).toHaveBeenCalledTimes(2);+ for (const [spec] of mocks.query.mock.calls) {+ expect(spec.query).toContain("FROM events");+ expect(spec.query).not.toContain("_mv_target");+ }+ expect(mocks.query.mock.calls[1][0].query).toContain("toStartOfMinute");+ });++ it("also uses exact events when only the comparison is a short rolling window", async () => {+ const response = await request({+ siteIds: [1],+ comparison: { past_minutes_start: 60, past_minutes_end: 30 },+ });+ expect(response.statusCode).toBe(200);+ expect(mocks.query.mock.calls[0][0].query).toContain("FROM events");+ });+it.each([{ siteIds: [], comparison },{ siteIds: [...allIds, 21], comparison },diff --git a/server/src/api/analytics/lite/getMetricLite.ts b/server/src/api/analytics/lite/getMetricLite.tsindex 688e05353..10c459447 100644--- a/server/src/api/analytics/lite/getMetricLite.ts+++ b/server/src/api/analytics/lite/getMetricLite.ts@@ -3,7 +3,7 @@ import { FastifyReply, FastifyRequest } from "fastify";import { getMetric } from "../getMetric.js";import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { getTimeStatement } from "../utils/timeWindow.js";-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters } from "./utils.js";+import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, hasLiteRealtimeRange } from "./utils.js";// Lite metric supports only dimensions backed by MVs:// - pathname → pathname_hourly_mv_target@@ -172,8 +172,8 @@ export const getMetricLite = analyticsRoute<GetMetricLiteRequest>(async (req: FastifyRequest<GetMetricLiteRequest>, res: FastifyReply) => {const site = Number(req.params.siteId);- // The hourly rollups can't express a sub-hour window.- if (hasLiteDatetimeRange(req.query)) {+ // Hourly rollups can't represent exact or short rolling windows.+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);}diff --git a/server/src/api/analytics/lite/getOverviewBucketedLite.ts b/server/src/api/analytics/lite/getOverviewBucketedLite.tsindex 4a8c20421..4065ca6bf 100644--- a/server/src/api/analytics/lite/getOverviewBucketedLite.ts+++ b/server/src/api/analytics/lite/getOverviewBucketedLite.ts@@ -4,7 +4,13 @@ import { getOverviewBucketed } from "../getOverviewBucketed.js";import { TimeBucket } from "../types.js";import { resolveTimeWindow } from "../utils/timeWindow.js";import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, liteBucket } from "./utils.js";+import {+ getLiteSessionFilter,+ hasLiteDatetimeRange,+ hasLiteFilters,+ hasLiteRealtimeRange,+ liteBucket,+} from "./utils.js";type GetOverviewBucketedLiteResponse = {time: string;@@ -76,11 +82,7 @@ function buildHourBucketQuery(args: {`;}-function buildDayBucketQuery(args: {- bucketed: (column: string) => string;- sessionTime: string;- fill: string;-}) {+function buildDayBucketQuery(args: { bucketed: (column: string) => string; sessionTime: string; fill: string }) {const { bucketed, sessionTime, fill } = args;// Aggregate in the inner GROUP BY, then compose ratios in the outer SELECT.// Aliasing `sum(sessions) AS sessions` would shadow the column inside the@@ -171,8 +173,8 @@ export const getOverviewBucketedLite = analyticsRoute<GetOverviewBucketedLiteReqasync (req: FastifyRequest<GetOverviewBucketedLiteRequest>, res: FastifyReply) => {const site = Number(req.params.siteId);- // The hourly rollups can't express a sub-hour window.- if (hasLiteDatetimeRange(req.query)) {+ // Hourly rollups can't represent exact or short rolling windows.+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {return getOverviewBucketed(req, res);}diff --git a/server/src/api/analytics/lite/getOverviewLite.ts b/server/src/api/analytics/lite/getOverviewLite.tsindex 3308479fb..e901a2c67 100644--- a/server/src/api/analytics/lite/getOverviewLite.ts+++ b/server/src/api/analytics/lite/getOverviewLite.ts@@ -3,7 +3,7 @@ import { FastifyReply, FastifyRequest } from "fastify";import { getOverview } from "../getOverview.js";import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { getTimeStatement } from "../utils/timeWindow.js";-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters } from "./utils.js";+import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, hasLiteRealtimeRange } from "./utils.js";type GetOverviewLiteResponse = {sessions: number;@@ -95,8 +95,8 @@ export const getOverviewLite = analyticsRoute<GetOverviewLiteRequest>(async (req: FastifyRequest<GetOverviewLiteRequest>, res: FastifyReply) => {const site = Number(req.params.siteId);- // The hourly rollups can't express a sub-hour window.- if (hasLiteDatetimeRange(req.query)) {+ // Hourly rollups can't represent exact or short rolling windows.+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {return getOverview(req, res);}diff --git a/server/src/api/analytics/lite/realtime.test.ts b/server/src/api/analytics/lite/realtime.test.tsnew file mode 100644index 000000000..8eaec1332--- /dev/null+++ b/server/src/api/analytics/lite/realtime.test.ts@@ -0,0 +1,46 @@+import { FilterParams, TimeBucket } from "@rybbit/shared";+import Fastify, { FastifyReply, FastifyRequest } from "fastify";+import { describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ overview: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: { sessions: 7 } })),+ chart: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: [{ sessions: 7 }] })),+ metric: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: { data: [{ count: 7 }] } })),+ query: vi.fn(),+}));+vi.mock("../getOverview.js", () => ({ getOverview: mocks.overview }));+vi.mock("../getOverviewBucketed.js", () => ({ getOverviewBucketed: mocks.chart }));+vi.mock("../getMetric.js", () => ({ getMetric: mocks.metric }));+vi.mock("../../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { query: mocks.query } }));+vi.mock("../../../db/postgres/postgres.js", () => ({ db: {} }));++import { getMetricLite } from "./getMetricLite.js";+import { getOverviewBucketedLite } from "./getOverviewBucketedLite.js";+import { getOverviewLite } from "./getOverviewLite.js";++describe("lite rolling-window routing", () => {+ it.each([+ { name: "overview", handler: getOverviewLite, raw: mocks.overview },+ { name: "chart", handler: getOverviewBucketedLite, raw: mocks.chart },+ { name: "metric", handler: getMetricLite, raw: mocks.metric },+ ])("uses exact event timestamps for the $name", async ({ handler, raw }) => {+ vi.clearAllMocks();+ const app = Fastify();+ app.get<{+ Params: { siteId: string };+ Querystring: FilterParams<{ bucket: TimeBucket; parameter: "pathname" }>;+ }>("/sites/:siteId/analytics", handler);+ try {+ for (const minutes of [5, 30, 60, 120]) {+ const response = await app.inject(+ `/sites/1/analytics?past_minutes_start=${minutes}&past_minutes_end=0&bucket=minute¶meter=pathname`+ );+ expect(response.statusCode).toBe(200);+ }+ expect(raw).toHaveBeenCalledTimes(4);+ expect(mocks.query).not.toHaveBeenCalled();+ } finally {+ await app.close();+ }+ });+});diff --git a/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.ts b/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.tsnew file mode 100644index 000000000..d84dd48e3--- /dev/null+++ b/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.ts@@ -0,0 +1,82 @@+import { createClient } from "@clickhouse/client";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { buildSiteCardsQueries } from "./siteCardsQuery.js";++// Read-only fixtures, following ../siteCardsQuery.clickhouse.test.ts. No tables+// or migrations are needed; the hourly snapshot intentionally trails events.+describe.skipIf(!process.env.CLICKHOUSE_TEST_URL)("lite Site cards: recent activity", () => {+ let client: ReturnType<typeof createClient>;+ const now = Date.parse("2026-09-21T14:07:56Z");+ const instant = (minutesAgo: number) =>+ new Date(now - minutesAgo * 60_000).toISOString().slice(0, 19).replace("T", " ");++ beforeAll(() => {+ client = createClient({+ url: process.env.CLICKHOUSE_TEST_URL,+ username: process.env.CLICKHOUSE_TEST_USER || "default",+ password: process.env.CLICKHOUSE_TEST_PASSWORD || "",+ clickhouse_settings: { readonly: "2", max_threads: 2, max_execution_time: 10 },+ });+ });+ afterAll(async () => {+ await client.close();+ });++ it.each([5, 30, 60, 120])("counts live sessions in a %i-minute window before the hourly refresh", async minutes => {+ const fixture = `WITH events AS (SELECT * FROM values(+ 'site_id UInt16, timestamp DateTime, session_id String, user_id String, identified_user_id String',+ (1, '${instant(minutes * 1.5)}', 'previous', 'old-user', ''),+ (1, '${instant(minutes)}', 'boundary', 'boundary-user', ''),+ (1, '${instant(minutes / 2)}', 'current-a', 'device-a', 'alice'),+ (1, '${instant(1)}', 'current-a', 'device-a', 'alice'),+ (1, '${instant(0.5)}', 'current-b', 'device-b', 'alice'),+ (2, '${instant(1)}', 'current-a', 'site-two', '')+ )), session_hourly_mv_target AS (+ SELECT site_id, toStartOfHour(timestamp) AS session_hour,+ uniqExact(session_id) AS sessions, uniqState(user_id) AS users+ FROM events WHERE timestamp < toStartOfHour(toDateTime('${instant(0)}', 'UTC'))+ GROUP BY site_id, session_hour+ ), sessions_mv_target AS (+ SELECT site_id, session_id, min(timestamp) AS start_time+ FROM events GROUP BY site_id, session_id+ )`;+ const queries = buildSiteCardsQueries(+ {+ siteIds: [1, 2, 3],+ current: { past_minutes_start: minutes, past_minutes_end: 0, time_zone: "UTC" },+ comparison: { past_minutes_start: minutes * 2, past_minutes_end: minutes, time_zone: "UTC" },+ bucket: "minute",+ },+ now+ );+ const totalsResult = await client.query({+ query: `${fixture} ${queries.totals.query}`,+ query_params: queries.totals.params,+ format: "JSONEachRow",+ });+ const totals = (await totalsResult.json<Record<string, number | string>>()).map(row =>+ Object.fromEntries(Object.entries(row).map(([key, value]) => [key, Number(value)]))+ );+ expect(totals.find(row => row.site_id === 1)).toMatchObject({+ current_sessions: 2,+ current_users: 1,+ previous_sessions: 2,+ previous_users: 2,+ });+ expect(totals.find(row => row.site_id === 2)).toMatchObject({ current_sessions: 1, current_users: 1 });+ expect(totals.find(row => row.site_id === 3)).toBeUndefined();++ const seriesResult = await client.query({+ query: `${fixture} ${queries.series.query}`,+ query_params: queries.series.params,+ format: "JSONEachRow",+ });+ const series = await seriesResult.json<{ time: string; site_sessions: [number, string][] }>();+ expect(series).toHaveLength(minutes + 1);+ const sessionsFor = (siteId: number) =>+ series.reduce((sum, row) => sum + Number(new Map(row.site_sessions).get(siteId) ?? 0), 0);+ expect(sessionsFor(1)).toBe(2);+ expect(sessionsFor(2)).toBe(1);+ expect(sessionsFor(3)).toBe(0);+ });+});diff --git a/server/src/api/analytics/lite/siteCardsQuery.ts b/server/src/api/analytics/lite/siteCardsQuery.tsindex 0c00f56d4..bda4758c9 100644--- a/server/src/api/analytics/lite/siteCardsQuery.ts+++ b/server/src/api/analytics/lite/siteCardsQuery.ts@@ -1,11 +1,19 @@-import { SiteCardQueries, SiteCardQueryParams } from "../siteCardsQuery.js";+import {+ buildSiteCardsQueries as buildRawSiteCardsQueries,+ SiteCardQueries,+ SiteCardQueryParams,+} from "../siteCardsQuery.js";import { resolveTimeWindow } from "../utils/timeWindow.js";-import { liteBucket } from "./utils.js";+import { hasLiteRealtimeRange, liteBucket } from "./utils.js";export function buildSiteCardsQueries({ siteIds, current, comparison, bucket: requestedBucket }: SiteCardQueryParams,now = Date.now()): SiteCardQueries {+ if (hasLiteRealtimeRange(current) || (comparison && hasLiteRealtimeRange(comparison))) {+ return buildRawSiteCardsQueries({ siteIds, current, comparison, bucket: requestedBucket }, now);+ }+// Resolve both periods against one clock, including adjacent rolling windows.const window = resolveTimeWindow(current, now);const previous = comparison === null ? null : resolveTimeWindow(comparison, now);diff --git a/server/src/api/analytics/lite/utils.test.ts b/server/src/api/analytics/lite/utils.test.tsindex 52cf36836..7e350e933 100644--- a/server/src/api/analytics/lite/utils.test.ts+++ b/server/src/api/analytics/lite/utils.test.ts@@ -5,6 +5,7 @@ import {getLiteSessionFilter,hasLiteDatetimeRange,hasLiteFilters,+ hasLiteRealtimeRange,liteBucket,wrapLiteLikeValue,} from "./utils.js";@@ -180,8 +181,9 @@ describe("hasLiteDatetimeRange", () => {// and the lite fill used to ignore these params outright, answering with// all-time data and a 200.it("should be true only when both datetime bounds are present", () => {- expect(hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00", end_datetime: "2024-01-01 14:45:00" })))- .toBe(true);+ expect(+ hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00", end_datetime: "2024-01-01 14:45:00" }))+ ).toBe(true);expect(hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00" }))).toBe(false);expect(hasLiteDatetimeRange(params({ end_datetime: "2024-01-01 14:45:00" }))).toBe(false);expect(hasLiteDatetimeRange(params({ start_date: "2024-01-01", end_date: "2024-01-31" }))).toBe(false);@@ -189,6 +191,26 @@ describe("hasLiteDatetimeRange", () => {});});+describe("hasLiteRealtimeRange", () => {+ it.each([+ [5, 0, true],+ [30, 0, true],+ [120, 0, true],+ [240, 120, true],+ [121, 0, false],+ [1440, 0, false],+ [2880, 1440, false],+ ])("uses the window length for %i to %i minutes ago", (start, end, expected) => {+ expect(hasLiteRealtimeRange(params({ past_minutes_start: start, past_minutes_end: end }))).toBe(expected);+ });++ it("accepts wire strings without treating missing bounds as realtime", () => {+ expect(hasLiteRealtimeRange({ past_minutes_start: "60", past_minutes_end: "30" })).toBe(true);+ expect(hasLiteRealtimeRange(params({ past_minutes_start: 30 }))).toBe(false);+ expect(hasLiteRealtimeRange(params({}))).toBe(false);+ });+});+describe("liteBucket", () => {it("should default undefined to hour", () => {expect(liteBucket(undefined)).toBe("hour");diff --git a/server/src/api/analytics/lite/utils.ts b/server/src/api/analytics/lite/utils.tsindex f09efce00..73b123e55 100644--- a/server/src/api/analytics/lite/utils.ts+++ b/server/src/api/analytics/lite/utils.ts@@ -104,8 +104,18 @@ export function liteBucket(bucket: TimeBucket | undefined): TimeBucket {// same escape hatch an unsupported filter takes. Before, both the lite time// statement and the lite fill simply ignored start_datetime/end_datetime, so the// request came back as all-time data with a 200.-export function hasLiteDatetimeRange(- params: Pick<FilterParams, "start_datetime" | "end_datetime">-): boolean {+export function hasLiteDatetimeRange(params: Pick<FilterParams, "start_datetime" | "end_datetime">): boolean {return Boolean(params.start_datetime && params.end_datetime);}++// Match the dashboard's minute-resolution rolling windows (up to two hours).+// Hourly timestamps can all fall before the cutoff, and the refreshable+// summary can omit the current hour entirely. Read exact events for these+// windows, including shifted comparison periods; keep long ranges on the MVs.+export function hasLiteRealtimeRange(params: {+ past_minutes_start?: number | string;+ past_minutes_end?: number | string;+}): boolean {+ const minutes = Number(params.past_minutes_start) - Number(params.past_minutes_end);+ return minutes > 0 && minutes <= 120;+}