Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

fix(analytics): read exact events for short rolling windows

Bill Yang7686128322bb
diff --git a/server/src/api/analytics/getSiteCards.test.ts b/server/src/api/analytics/getSiteCards.test.ts
index 935bbf6c5..1390296f5 100644
--- a/server/src/api/analytics/getSiteCards.test.ts
+++ b/server/src/api/analytics/getSiteCards.test.ts
@@ -123,6 +123,32 @@ describe.each(["site-cards", "site-cards-lite"])("batched %s", endpoint => {
expect(mocks.query.mock.calls[0][0].query).not.toContain("2026-09-18");
});
+ it.each([5, 30, 60, 120])("reads exact events for a %i-minute rolling window", async minutes => {
+ const response = await request(
+ {
+ siteIds: [1, 2],
+ comparison: { past_minutes_start: minutes * 2, past_minutes_end: minutes },
+ },
+ `/organizations/org-1/${endpoint}?past_minutes_start=${minutes}&past_minutes_end=0&bucket=minute`
+ );
+ expect(response.statusCode).toBe(200);
+ expect(mocks.query).toHaveBeenCalledTimes(2);
+ for (const [spec] of mocks.query.mock.calls) {
+ expect(spec.query).toContain("FROM events");
+ expect(spec.query).not.toContain("_mv_target");
+ }
+ expect(mocks.query.mock.calls[1][0].query).toContain("toStartOfMinute");
+ });
+
+ it("also uses exact events when only the comparison is a short rolling window", async () => {
+ const response = await request({
+ siteIds: [1],
+ comparison: { past_minutes_start: 60, past_minutes_end: 30 },
+ });
+ expect(response.statusCode).toBe(200);
+ expect(mocks.query.mock.calls[0][0].query).toContain("FROM events");
+ });
+
it.each([
{ siteIds: [], comparison },
{ siteIds: [...allIds, 21], comparison },
diff --git a/server/src/api/analytics/lite/getMetricLite.ts b/server/src/api/analytics/lite/getMetricLite.ts
index 688e05353..10c459447 100644
--- a/server/src/api/analytics/lite/getMetricLite.ts
+++ b/server/src/api/analytics/lite/getMetricLite.ts
@@ -3,7 +3,7 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { getMetric } from "../getMetric.js";
import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { getTimeStatement } from "../utils/timeWindow.js";
-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters } from "./utils.js";
+import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, hasLiteRealtimeRange } from "./utils.js";
// Lite metric supports only dimensions backed by MVs:
// - pathname → pathname_hourly_mv_target
@@ -172,8 +172,8 @@ export const getMetricLite = analyticsRoute<GetMetricLiteRequest>(
async (req: FastifyRequest<GetMetricLiteRequest>, res: FastifyReply) => {
const site = Number(req.params.siteId);
- // The hourly rollups can't express a sub-hour window.
- if (hasLiteDatetimeRange(req.query)) {
+ // Hourly rollups can't represent exact or short rolling windows.
+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {
return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);
}
diff --git a/server/src/api/analytics/lite/getOverviewBucketedLite.ts b/server/src/api/analytics/lite/getOverviewBucketedLite.ts
index 4a8c20421..4065ca6bf 100644
--- a/server/src/api/analytics/lite/getOverviewBucketedLite.ts
+++ b/server/src/api/analytics/lite/getOverviewBucketedLite.ts
@@ -4,7 +4,13 @@ import { getOverviewBucketed } from "../getOverviewBucketed.js";
import { TimeBucket } from "../types.js";
import { resolveTimeWindow } from "../utils/timeWindow.js";
import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, liteBucket } from "./utils.js";
+import {
+ getLiteSessionFilter,
+ hasLiteDatetimeRange,
+ hasLiteFilters,
+ hasLiteRealtimeRange,
+ liteBucket,
+} from "./utils.js";
type GetOverviewBucketedLiteResponse = {
time: string;
@@ -76,11 +82,7 @@ function buildHourBucketQuery(args: {
`;
}
-function buildDayBucketQuery(args: {
- bucketed: (column: string) => string;
- sessionTime: string;
- fill: string;
-}) {
+function buildDayBucketQuery(args: { bucketed: (column: string) => string; sessionTime: string; fill: string }) {
const { bucketed, sessionTime, fill } = args;
// Aggregate in the inner GROUP BY, then compose ratios in the outer SELECT.
// Aliasing `sum(sessions) AS sessions` would shadow the column inside the
@@ -171,8 +173,8 @@ export const getOverviewBucketedLite = analyticsRoute<GetOverviewBucketedLiteReq
async (req: FastifyRequest<GetOverviewBucketedLiteRequest>, res: FastifyReply) => {
const site = Number(req.params.siteId);
- // The hourly rollups can't express a sub-hour window.
- if (hasLiteDatetimeRange(req.query)) {
+ // Hourly rollups can't represent exact or short rolling windows.
+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {
return getOverviewBucketed(req, res);
}
diff --git a/server/src/api/analytics/lite/getOverviewLite.ts b/server/src/api/analytics/lite/getOverviewLite.ts
index 3308479fb..e901a2c67 100644
--- a/server/src/api/analytics/lite/getOverviewLite.ts
+++ b/server/src/api/analytics/lite/getOverviewLite.ts
@@ -3,7 +3,7 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { getOverview } from "../getOverview.js";
import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { getTimeStatement } from "../utils/timeWindow.js";
-import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters } from "./utils.js";
+import { getLiteSessionFilter, hasLiteDatetimeRange, hasLiteFilters, hasLiteRealtimeRange } from "./utils.js";
type GetOverviewLiteResponse = {
sessions: number;
@@ -95,8 +95,8 @@ export const getOverviewLite = analyticsRoute<GetOverviewLiteRequest>(
async (req: FastifyRequest<GetOverviewLiteRequest>, res: FastifyReply) => {
const site = Number(req.params.siteId);
- // The hourly rollups can't express a sub-hour window.
- if (hasLiteDatetimeRange(req.query)) {
+ // Hourly rollups can't represent exact or short rolling windows.
+ if (hasLiteDatetimeRange(req.query) || hasLiteRealtimeRange(req.query)) {
return getOverview(req, res);
}
diff --git a/server/src/api/analytics/lite/realtime.test.ts b/server/src/api/analytics/lite/realtime.test.ts
new file mode 100644
index 000000000..8eaec1332
--- /dev/null
+++ b/server/src/api/analytics/lite/realtime.test.ts
@@ -0,0 +1,46 @@
+import { FilterParams, TimeBucket } from "@rybbit/shared";
+import Fastify, { FastifyReply, FastifyRequest } from "fastify";
+import { describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ overview: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: { sessions: 7 } })),
+ chart: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: [{ sessions: 7 }] })),
+ metric: vi.fn(async (_req: FastifyRequest, res: FastifyReply) => res.send({ data: { data: [{ count: 7 }] } })),
+ query: vi.fn(),
+}));
+vi.mock("../getOverview.js", () => ({ getOverview: mocks.overview }));
+vi.mock("../getOverviewBucketed.js", () => ({ getOverviewBucketed: mocks.chart }));
+vi.mock("../getMetric.js", () => ({ getMetric: mocks.metric }));
+vi.mock("../../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { query: mocks.query } }));
+vi.mock("../../../db/postgres/postgres.js", () => ({ db: {} }));
+
+import { getMetricLite } from "./getMetricLite.js";
+import { getOverviewBucketedLite } from "./getOverviewBucketedLite.js";
+import { getOverviewLite } from "./getOverviewLite.js";
+
+describe("lite rolling-window routing", () => {
+ it.each([
+ { name: "overview", handler: getOverviewLite, raw: mocks.overview },
+ { name: "chart", handler: getOverviewBucketedLite, raw: mocks.chart },
+ { name: "metric", handler: getMetricLite, raw: mocks.metric },
+ ])("uses exact event timestamps for the $name", async ({ handler, raw }) => {
+ vi.clearAllMocks();
+ const app = Fastify();
+ app.get<{
+ Params: { siteId: string };
+ Querystring: FilterParams<{ bucket: TimeBucket; parameter: "pathname" }>;
+ }>("/sites/:siteId/analytics", handler);
+ try {
+ for (const minutes of [5, 30, 60, 120]) {
+ const response = await app.inject(
+ `/sites/1/analytics?past_minutes_start=${minutes}&past_minutes_end=0&bucket=minute&parameter=pathname`
+ );
+ expect(response.statusCode).toBe(200);
+ }
+ expect(raw).toHaveBeenCalledTimes(4);
+ expect(mocks.query).not.toHaveBeenCalled();
+ } finally {
+ await app.close();
+ }
+ });
+});
diff --git a/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.ts b/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.ts
new file mode 100644
index 000000000..d84dd48e3
--- /dev/null
+++ b/server/src/api/analytics/lite/siteCardsQuery.clickhouse.test.ts
@@ -0,0 +1,82 @@
+import { createClient } from "@clickhouse/client";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { buildSiteCardsQueries } from "./siteCardsQuery.js";
+
+// Read-only fixtures, following ../siteCardsQuery.clickhouse.test.ts. No tables
+// or migrations are needed; the hourly snapshot intentionally trails events.
+describe.skipIf(!process.env.CLICKHOUSE_TEST_URL)("lite Site cards: recent activity", () => {
+ let client: ReturnType<typeof createClient>;
+ const now = Date.parse("2026-09-21T14:07:56Z");
+ const instant = (minutesAgo: number) =>
+ new Date(now - minutesAgo * 60_000).toISOString().slice(0, 19).replace("T", " ");
+
+ beforeAll(() => {
+ client = createClient({
+ url: process.env.CLICKHOUSE_TEST_URL,
+ username: process.env.CLICKHOUSE_TEST_USER || "default",
+ password: process.env.CLICKHOUSE_TEST_PASSWORD || "",
+ clickhouse_settings: { readonly: "2", max_threads: 2, max_execution_time: 10 },
+ });
+ });
+ afterAll(async () => {
+ await client.close();
+ });
+
+ it.each([5, 30, 60, 120])("counts live sessions in a %i-minute window before the hourly refresh", async minutes => {
+ const fixture = `WITH events AS (SELECT * FROM values(
+ 'site_id UInt16, timestamp DateTime, session_id String, user_id String, identified_user_id String',
+ (1, '${instant(minutes * 1.5)}', 'previous', 'old-user', ''),
+ (1, '${instant(minutes)}', 'boundary', 'boundary-user', ''),
+ (1, '${instant(minutes / 2)}', 'current-a', 'device-a', 'alice'),
+ (1, '${instant(1)}', 'current-a', 'device-a', 'alice'),
+ (1, '${instant(0.5)}', 'current-b', 'device-b', 'alice'),
+ (2, '${instant(1)}', 'current-a', 'site-two', '')
+ )), session_hourly_mv_target AS (
+ SELECT site_id, toStartOfHour(timestamp) AS session_hour,
+ uniqExact(session_id) AS sessions, uniqState(user_id) AS users
+ FROM events WHERE timestamp < toStartOfHour(toDateTime('${instant(0)}', 'UTC'))
+ GROUP BY site_id, session_hour
+ ), sessions_mv_target AS (
+ SELECT site_id, session_id, min(timestamp) AS start_time
+ FROM events GROUP BY site_id, session_id
+ )`;
+ const queries = buildSiteCardsQueries(
+ {
+ siteIds: [1, 2, 3],
+ current: { past_minutes_start: minutes, past_minutes_end: 0, time_zone: "UTC" },
+ comparison: { past_minutes_start: minutes * 2, past_minutes_end: minutes, time_zone: "UTC" },
+ bucket: "minute",
+ },
+ now
+ );
+ const totalsResult = await client.query({
+ query: `${fixture} ${queries.totals.query}`,
+ query_params: queries.totals.params,
+ format: "JSONEachRow",
+ });
+ const totals = (await totalsResult.json<Record<string, number | string>>()).map(row =>
+ Object.fromEntries(Object.entries(row).map(([key, value]) => [key, Number(value)]))
+ );
+ expect(totals.find(row => row.site_id === 1)).toMatchObject({
+ current_sessions: 2,
+ current_users: 1,
+ previous_sessions: 2,
+ previous_users: 2,
+ });
+ expect(totals.find(row => row.site_id === 2)).toMatchObject({ current_sessions: 1, current_users: 1 });
+ expect(totals.find(row => row.site_id === 3)).toBeUndefined();
+
+ const seriesResult = await client.query({
+ query: `${fixture} ${queries.series.query}`,
+ query_params: queries.series.params,
+ format: "JSONEachRow",
+ });
+ const series = await seriesResult.json<{ time: string; site_sessions: [number, string][] }>();
+ expect(series).toHaveLength(minutes + 1);
+ const sessionsFor = (siteId: number) =>
+ series.reduce((sum, row) => sum + Number(new Map(row.site_sessions).get(siteId) ?? 0), 0);
+ expect(sessionsFor(1)).toBe(2);
+ expect(sessionsFor(2)).toBe(1);
+ expect(sessionsFor(3)).toBe(0);
+ });
+});
diff --git a/server/src/api/analytics/lite/siteCardsQuery.ts b/server/src/api/analytics/lite/siteCardsQuery.ts
index 0c00f56d4..bda4758c9 100644
--- a/server/src/api/analytics/lite/siteCardsQuery.ts
+++ b/server/src/api/analytics/lite/siteCardsQuery.ts
@@ -1,11 +1,19 @@
-import { SiteCardQueries, SiteCardQueryParams } from "../siteCardsQuery.js";
+import {
+ buildSiteCardsQueries as buildRawSiteCardsQueries,
+ SiteCardQueries,
+ SiteCardQueryParams,
+} from "../siteCardsQuery.js";
import { resolveTimeWindow } from "../utils/timeWindow.js";
-import { liteBucket } from "./utils.js";
+import { hasLiteRealtimeRange, liteBucket } from "./utils.js";
export function buildSiteCardsQueries(
{ siteIds, current, comparison, bucket: requestedBucket }: SiteCardQueryParams,
now = Date.now()
): SiteCardQueries {
+ if (hasLiteRealtimeRange(current) || (comparison && hasLiteRealtimeRange(comparison))) {
+ return buildRawSiteCardsQueries({ siteIds, current, comparison, bucket: requestedBucket }, now);
+ }
+
// Resolve both periods against one clock, including adjacent rolling windows.
const window = resolveTimeWindow(current, now);
const previous = comparison === null ? null : resolveTimeWindow(comparison, now);
diff --git a/server/src/api/analytics/lite/utils.test.ts b/server/src/api/analytics/lite/utils.test.ts
index 52cf36836..7e350e933 100644
--- a/server/src/api/analytics/lite/utils.test.ts
+++ b/server/src/api/analytics/lite/utils.test.ts
@@ -5,6 +5,7 @@ import {
getLiteSessionFilter,
hasLiteDatetimeRange,
hasLiteFilters,
+ hasLiteRealtimeRange,
liteBucket,
wrapLiteLikeValue,
} from "./utils.js";
@@ -180,8 +181,9 @@ describe("hasLiteDatetimeRange", () => {
// and the lite fill used to ignore these params outright, answering with
// all-time data and a 200.
it("should be true only when both datetime bounds are present", () => {
- expect(hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00", end_datetime: "2024-01-01 14:45:00" })))
- .toBe(true);
+ expect(
+ hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00", end_datetime: "2024-01-01 14:45:00" }))
+ ).toBe(true);
expect(hasLiteDatetimeRange(params({ start_datetime: "2024-01-01 10:30:00" }))).toBe(false);
expect(hasLiteDatetimeRange(params({ end_datetime: "2024-01-01 14:45:00" }))).toBe(false);
expect(hasLiteDatetimeRange(params({ start_date: "2024-01-01", end_date: "2024-01-31" }))).toBe(false);
@@ -189,6 +191,26 @@ describe("hasLiteDatetimeRange", () => {
});
});
+describe("hasLiteRealtimeRange", () => {
+ it.each([
+ [5, 0, true],
+ [30, 0, true],
+ [120, 0, true],
+ [240, 120, true],
+ [121, 0, false],
+ [1440, 0, false],
+ [2880, 1440, false],
+ ])("uses the window length for %i to %i minutes ago", (start, end, expected) => {
+ expect(hasLiteRealtimeRange(params({ past_minutes_start: start, past_minutes_end: end }))).toBe(expected);
+ });
+
+ it("accepts wire strings without treating missing bounds as realtime", () => {
+ expect(hasLiteRealtimeRange({ past_minutes_start: "60", past_minutes_end: "30" })).toBe(true);
+ expect(hasLiteRealtimeRange(params({ past_minutes_start: 30 }))).toBe(false);
+ expect(hasLiteRealtimeRange(params({}))).toBe(false);
+ });
+});
+
describe("liteBucket", () => {
it("should default undefined to hour", () => {
expect(liteBucket(undefined)).toBe("hour");
diff --git a/server/src/api/analytics/lite/utils.ts b/server/src/api/analytics/lite/utils.ts
index f09efce00..73b123e55 100644
--- a/server/src/api/analytics/lite/utils.ts
+++ b/server/src/api/analytics/lite/utils.ts
@@ -104,8 +104,18 @@ export function liteBucket(bucket: TimeBucket | undefined): TimeBucket {
// same escape hatch an unsupported filter takes. Before, both the lite time
// statement and the lite fill simply ignored start_datetime/end_datetime, so the
// request came back as all-time data with a 200.
-export function hasLiteDatetimeRange(
- params: Pick<FilterParams, "start_datetime" | "end_datetime">
-): boolean {
+export function hasLiteDatetimeRange(params: Pick<FilterParams, "start_datetime" | "end_datetime">): boolean {
return Boolean(params.start_datetime && params.end_datetime);
}
+
+// Match the dashboard's minute-resolution rolling windows (up to two hours).
+// Hourly timestamps can all fall before the cutoff, and the refreshable
+// summary can omit the current hour entirely. Read exact events for these
+// windows, including shifted comparison periods; keep long ranges on the MVs.
+export function hasLiteRealtimeRange(params: {
+ past_minutes_start?: number | string;
+ past_minutes_end?: number | string;
+}): boolean {
+ const minutes = Number(params.past_minutes_start) - Number(params.past_minutes_end);
+ return minutes > 0 && minutes <= 120;
+}