Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor analytics API endpoints to improve query handling and error management

Bill Yang66b49e9925cc
diff --git a/server/src/api/analytics/bots/getBotDimension.ts b/server/src/api/analytics/bots/getBotDimension.ts
index 84dc7ca7b..63f5466d7 100644
--- a/server/src/api/analytics/bots/getBotDimension.ts
+++ b/server/src/api/analytics/bots/getBotDimension.ts
@@ -1,7 +1,7 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "../utils/analyticsQuery.js";
import {
BOT_DIMENSIONS,
type BotDimensionKey,
@@ -35,24 +35,17 @@ export interface BotDimensionRequest {
}>;
}
-const parsePositiveInt = (value: unknown, fallback: number) => {
- const parsed = parseInt(String(value), 10);
- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-};
-
-const getQuery = (request: FastifyRequest<BotDimensionRequest>, isCountQuery = false) => {
- const { dimension, limit, page } = request.query;
+export const buildBotDimensionQuery = (query: BotDimensionRequest["Querystring"], isCountQuery = false) => {
+ const { dimension } = query;
if (!BOT_DIMENSIONS.has(dimension)) {
throw new Error(`Unsupported bot dimension: ${dimension}`);
}
const expression = getBotSqlParam(dimension);
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getBotFilterStatement(request.query.filters);
- const layerStatement = getBotLayerStatement(request.query.layer);
- const validatedLimit = parsePositiveInt(limit, 100);
- const validatedPage = parsePositiveInt(page, 1);
- const offset = (validatedPage - 1) * validatedLimit;
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getBotFilterStatement(query.filters);
+ const layerStatement = getBotLayerStatement(query.layer);
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);
const groupedQuery = `
SELECT
@@ -78,40 +71,21 @@ const getQuery = (request: FastifyRequest<BotDimensionRequest>, isCountQuery = f
return `
${groupedQuery}
ORDER BY count DESC
- LIMIT ${validatedLimit}
- OFFSET ${offset}
+ ${limitStatement}
+ ${offsetStatement}
`;
};
-export async function getBotDimension(req: FastifyRequest<BotDimensionRequest>, res: FastifyReply) {
- try {
- const [dataResult, countResult] = await Promise.all([
- clickhouse.query({
- query: getQuery(req),
- format: "JSONEachRow",
- query_params: {
- siteId: Number(req.params.siteId),
- },
- }),
- clickhouse.query({
- query: getQuery(req, true),
- format: "JSONEachRow",
- query_params: {
- siteId: Number(req.params.siteId),
- },
- }),
- ]);
+export const getBotDimension = analyticsRoute<BotDimensionRequest>(
+ "bot dimension",
+ async (req: FastifyRequest<BotDimensionRequest>, res: FastifyReply) => {
+ const params = { siteId: Number(req.params.siteId) };
- const data = await processResults<BotDimensionItem>(dataResult);
- const countData = await processResults<{ totalCount: number }>(countResult);
- const response: BotDimensionResponse = {
- data,
- totalCount: countData[0]?.totalCount ?? 0,
- };
+ const response: BotDimensionResponse = await runPaginatedQuery<BotDimensionItem>(
+ { query: buildBotDimensionQuery(req.query), params },
+ { query: buildBotDimensionQuery(req.query, true), params }
+ );
return res.send({ data: response });
- } catch (error) {
- console.error("Error fetching bot dimension:", error);
- return res.status(500).send({ error: "Failed to fetch bot dimension" });
}
-}
+);
diff --git a/server/src/api/analytics/bots/getBotOverview.ts b/server/src/api/analytics/bots/getBotOverview.ts
index ca59204d9..fe8fda674 100644
--- a/server/src/api/analytics/bots/getBotOverview.ts
+++ b/server/src/api/analytics/bots/getBotOverview.ts
@@ -1,7 +1,7 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement } from "./utils.js";
type BotOverviewResponse = {
@@ -24,10 +24,10 @@ export interface BotOverviewRequest {
}>;
}
-const getQuery = (params: BotOverviewRequest["Querystring"]) => {
- const timeStatement = getTimeStatement(params);
- const filterStatement = getBotFilterStatement(params.filters);
- const layerStatement = getBotLayerStatement(params.layer);
+export const buildBotOverviewQuery = (query: BotOverviewRequest["Querystring"]) => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getBotFilterStatement(query.filters);
+ const layerStatement = getBotLayerStatement(query.layer);
return `
WITH
@@ -78,20 +78,14 @@ const getQuery = (params: BotOverviewRequest["Querystring"]) => {
`;
};
-export async function getBotOverview(req: FastifyRequest<BotOverviewRequest>, res: FastifyReply) {
- try {
- const result = await clickhouse.query({
- query: getQuery(req.query),
- format: "JSONEachRow",
- query_params: {
- siteId: Number(req.params.siteId),
- },
+export const getBotOverview = analyticsRoute<BotOverviewRequest>(
+ "bot overview",
+ async (req: FastifyRequest<BotOverviewRequest>, res: FastifyReply) => {
+ const data = await runAnalyticsQuery<BotOverviewResponse>({
+ query: buildBotOverviewQuery(req.query),
+ params: { siteId: Number(req.params.siteId) },
});
- const data = await processResults<BotOverviewResponse>(result);
return res.send({ data: data[0] });
- } catch (error) {
- console.error("Error fetching bot overview:", error);
- return res.status(500).send({ error: "Failed to fetch bot overview" });
}
-}
+);
diff --git a/server/src/api/analytics/bots/getBotTimeSeries.ts b/server/src/api/analytics/bots/getBotTimeSeries.ts
index 2376fba6a..33df5bf84 100644
--- a/server/src/api/analytics/bots/getBotTimeSeries.ts
+++ b/server/src/api/analytics/bots/getBotTimeSeries.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { TimeBucket } from "../types.js";
-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";
+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement, getBotTimeStatementFill } from "./utils.js";
type BotTimeSeriesPoint = {
@@ -21,16 +21,16 @@ export interface BotTimeSeriesRequest {
}>;
}
-const getQuery = (params: BotTimeSeriesRequest["Querystring"]) => {
- const { bucket = "hour", time_zone } = params;
- const timeStatement = getTimeStatement(params);
- const filterStatement = getBotFilterStatement(params.filters);
- const layerStatement = getBotLayerStatement(params.layer);
+export const buildBotTimeSeriesQuery = (query: BotTimeSeriesRequest["Querystring"]) => {
+ const { bucket = "hour", time_zone } = query;
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getBotFilterStatement(query.filters);
+ const layerStatement = getBotLayerStatement(query.layer);
const hasBoundedTime =
- Boolean(params.start_date && params.end_date) ||
- Boolean(params.start_datetime && params.end_datetime) ||
- (params.past_minutes_start !== undefined && params.past_minutes_end !== undefined);
- const fillClause = hasBoundedTime ? getBotTimeStatementFill(params, bucket) : "";
+ Boolean(query.start_date && query.end_date) ||
+ Boolean(query.start_datetime && query.end_datetime) ||
+ (query.past_minutes_start !== undefined && query.past_minutes_end !== undefined);
+ const fillClause = hasBoundedTime ? getBotTimeStatementFill(query, bucket) : "";
const timezone = SqlString.escape(time_zone || "UTC");
return `
@@ -47,20 +47,14 @@ const getQuery = (params: BotTimeSeriesRequest["Querystring"]) => {
`;
};
-export async function getBotTimeSeries(req: FastifyRequest<BotTimeSeriesRequest>, res: FastifyReply) {
- try {
- const result = await clickhouse.query({
- query: getQuery(req.query),
- format: "JSONEachRow",
- query_params: {
- siteId: Number(req.params.siteId),
- },
+export const getBotTimeSeries = analyticsRoute<BotTimeSeriesRequest>(
+ "bot time series",
+ async (req: FastifyRequest<BotTimeSeriesRequest>, res: FastifyReply) => {
+ const data = await runAnalyticsQuery<BotTimeSeriesPoint>({
+ query: buildBotTimeSeriesQuery(req.query),
+ params: { siteId: Number(req.params.siteId) },
});
- const data = await processResults<BotTimeSeriesPoint>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching bot time series:", error);
- return res.status(500).send({ error: "Failed to fetch bot time series" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getAutocaptureEvents.ts b/server/src/api/analytics/events/getAutocaptureEvents.ts
index ee0effad8..82c512605 100644
--- a/server/src/api/analytics/events/getAutocaptureEvents.ts
+++ b/server/src/api/analytics/events/getAutocaptureEvents.ts
@@ -1,10 +1,10 @@
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
import { AutocaptureTargetType, isAutocaptureTargetType } from "../utils/eventConditions.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetAutocaptureEventsResponse = {
value: string;
@@ -33,21 +33,18 @@ const VALUE_EXPRESSIONS: Record<AutocaptureTargetType, string> = {
"coalesce(nullIf(JSONExtractString(toString(props), 'formName'), ''), nullIf(JSONExtractString(toString(props), 'formId'), ''), nullIf(JSONExtractString(toString(props), 'formAction'), ''), '')",
};
-// Returns autocapture events of a type (button clicks, form submissions,
-// copies) grouped by their display value, with counts and last occurrence.
-export async function getAutocaptureEvents(req: FastifyRequest<GetAutocaptureEventsRequest>, res: FastifyReply) {
- const { type, filters } = req.query;
- const site = req.params.siteId;
+export const buildAutocaptureEventsQuery = (
+ query: GetAutocaptureEventsRequest["Querystring"],
+ siteId: number,
+ type: AutocaptureTargetType
+) => {
+ const { filters } = query;
- if (!type || !isAutocaptureTargetType(type)) {
- return res.status(400).send({ error: "Invalid autocapture event type" });
- }
-
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";
const valueExpression = VALUE_EXPRESSIONS[type];
- const query = `
+ return `
SELECT
${valueExpression} AS value,
COUNT(*) AS count,
@@ -63,23 +60,27 @@ export async function getAutocaptureEvents(req: FastifyRequest<GetAutocaptureEve
ORDER BY count DESC
LIMIT 1000
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+// Returns autocapture events of a type (button clicks, form submissions,
+// copies) grouped by their display value, with counts and last occurrence.
+export const getAutocaptureEvents = analyticsRoute<GetAutocaptureEventsRequest>(
+ "autocapture events",
+ async (req: FastifyRequest<GetAutocaptureEventsRequest>, res: FastifyReply) => {
+ const { type } = req.query;
+ const site = req.params.siteId;
+
+ if (!type || !isAutocaptureTargetType(type)) {
+ return res.status(400).send({ error: "Invalid autocapture event type" });
+ }
+
+ const data = await runAnalyticsQuery<GetAutocaptureEventsResponse[number]>({
+ query: buildAutocaptureEventsQuery(req.query, Number(site), type),
+ params: { siteId: Number(site) },
});
- const data = await processResults<GetAutocaptureEventsResponse[number]>(result);
// processResults coerces any column that looks numeric into a number; `value`
// is free-form captured text (e.g. a button labeled "100") and must stay a string.
return res.send({ data: data.map(row => ({ ...row, value: String(row.value) })) });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching autocapture events:", error);
- return res.status(500).send({ error: "Failed to fetch autocapture events" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getAutocaptureValues.ts b/server/src/api/analytics/events/getAutocaptureValues.ts
index 03d515c71..46397cfff 100644
--- a/server/src/api/analytics/events/getAutocaptureValues.ts
+++ b/server/src/api/analytics/events/getAutocaptureValues.ts
@@ -1,10 +1,10 @@
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
import { FilterParams } from "@rybbit/shared";
-import { AUTOCAPTURE_PATTERN_PROPS, isAutocaptureTargetType } from "../utils/eventConditions.js";
+import { AUTOCAPTURE_PATTERN_PROPS, AutocaptureTargetType, isAutocaptureTargetType } from "../utils/eventConditions.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetAutocaptureValuesResponse = {
value: string;
@@ -20,25 +20,21 @@ export interface GetAutocaptureValuesRequest {
}>;
}
-// Returns the most common values of an autocapture type's primary props
-// (outbound urls, button texts, form names/ids, copied texts), used as
-// suggestions when configuring goals and funnel steps.
-export async function getAutocaptureValues(req: FastifyRequest<GetAutocaptureValuesRequest>, res: FastifyReply) {
- const { type, filters } = req.query;
- const site = req.params.siteId;
+export const buildAutocaptureValuesQuery = (
+ query: GetAutocaptureValuesRequest["Querystring"],
+ siteId: number,
+ type: AutocaptureTargetType
+) => {
+ const { filters } = query;
- if (!type || !isAutocaptureTargetType(type)) {
- return res.status(400).send({ error: "Invalid autocapture event type" });
- }
-
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";
const propExtracts = AUTOCAPTURE_PATTERN_PROPS[type]
.map(prop => `JSONExtractString(toString(props), ${SqlString.escape(prop)})`)
.join(", ");
- const query = `
+ return `
SELECT value, COUNT(*) AS count
FROM (
SELECT arrayJoin([${propExtracts}]) AS value
@@ -54,22 +50,28 @@ export async function getAutocaptureValues(req: FastifyRequest<GetAutocaptureVal
ORDER BY count DESC
LIMIT 500
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+// Returns the most common values of an autocapture type's primary props
+// (outbound urls, button texts, form names/ids, copied texts), used as
+// suggestions when configuring goals and funnel steps.
+export const getAutocaptureValues = analyticsRoute<GetAutocaptureValuesRequest>(
+ "autocapture values",
+ async (req: FastifyRequest<GetAutocaptureValuesRequest>, res: FastifyReply) => {
+ const { type } = req.query;
+ const site = req.params.siteId;
+
+ if (!type || !isAutocaptureTargetType(type)) {
+ return res.status(400).send({ error: "Invalid autocapture event type" });
+ }
+
+ const data = await runAnalyticsQuery<GetAutocaptureValuesResponse[number]>({
+ query: buildAutocaptureValuesQuery(req.query, Number(site), type),
+ params: { siteId: Number(site) },
});
- const data = await processResults<GetAutocaptureValuesResponse[number]>(result);
// processResults coerces any column that looks numeric into a number; `value`
// is free-form captured text (e.g. a button labeled "100") and must stay a string.
return res.send({ data: data.map(row => ({ ...row, value: String(row.value) })) });
- } catch (error) {
- console.error("Error fetching autocapture values:", error);
- return res.status(500).send({ error: "Failed to fetch autocapture values" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getEventBucketed.ts b/server/src/api/analytics/events/getEventBucketed.ts
index 7f956cd24..2f3980d13 100644
--- a/server/src/api/analytics/events/getEventBucketed.ts
+++ b/server/src/api/analytics/events/getEventBucketed.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { TimeBucket } from "../types.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";
+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetEventBucketedResponse = {
time: string;
@@ -29,21 +29,13 @@ const parseLimit = (limit?: string) => {
return Math.min(Math.max(Math.floor(parsed), 1), 10);
};
-export async function getEventBucketed(req: FastifyRequest<GetEventBucketedRequest>, res: FastifyReply) {
- const site = req.params.siteId;
- const { bucket = "hour", limit } = req.query;
- const timeZone = req.query.time_zone || "UTC";
+export const buildEventBucketedQuery = (query: GetEventBucketedRequest["Querystring"], siteId: number) => {
+ const { bucket = "hour" } = query;
- if (!TimeBucketToFn[bucket]) {
- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
- }
-
- const topLimit = parseLimit(limit);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(req.query.filters, Number(site), timeStatement);
-
- const query = `
+ return `
WITH top_events AS (
SELECT
event_name
@@ -75,23 +67,28 @@ export async function getEventBucketed(req: FastifyRequest<GetEventBucketedReque
GROUP BY time, event_name
ORDER BY time
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+export const getEventBucketed = analyticsRoute<GetEventBucketedRequest>(
+ "bucketed event data",
+ async (req: FastifyRequest<GetEventBucketedRequest>, res: FastifyReply) => {
+ const site = req.params.siteId;
+ const { bucket = "hour", limit } = req.query;
+ const timeZone = req.query.time_zone || "UTC";
+
+ if (!TimeBucketToFn[bucket]) {
+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
+ }
+
+ const data = await runAnalyticsQuery<GetEventBucketedResponse[number]>({
+ query: buildEventBucketedQuery(req.query, Number(site)),
+ params: {
siteId: Number(site),
- limit: topLimit,
+ limit: parseLimit(limit),
timeZone,
},
});
- const data = await processResults<GetEventBucketedResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching bucketed event data:", error);
- return res.status(500).send({ error: "Failed to fetch bucketed event data" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getEventNames.ts b/server/src/api/analytics/events/getEventNames.ts
index 829f80486..f9e9a90ca 100644
--- a/server/src/api/analytics/events/getEventNames.ts
+++ b/server/src/api/analytics/events/getEventNames.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetEventNamesResponse = {
eventName: string;
@@ -18,15 +18,14 @@ export interface GetEventNamesRequest {
}>;
}
-export async function getEventNames(req: FastifyRequest<GetEventNamesRequest>, res: FastifyReply) {
- const { filters } = req.query;
- const site = req.params.siteId;
+export const buildEventNamesQuery = (query: GetEventNamesRequest["Querystring"], siteId: number) => {
+ const { filters } = query;
- const timeStatement = getTimeStatement(req.query);
+ const timeStatement = getTimeStatement(query);
- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";
+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";
- const query = `
+ return `
SELECT
event_name AS eventName,
count() AS count
@@ -42,21 +41,18 @@ export async function getEventNames(req: FastifyRequest<GetEventNamesRequest>, r
ORDER BY count DESC
LIMIT 1000
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+export const getEventNames = analyticsRoute<GetEventNamesRequest>(
+ "event names",
+ async (req: FastifyRequest<GetEventNamesRequest>, res: FastifyReply) => {
+ const site = req.params.siteId;
+
+ const data = await runAnalyticsQuery<GetEventNamesResponse[number]>({
+ query: buildEventNamesQuery(req.query, Number(site)),
+ params: { siteId: Number(site) },
});
- const data = await processResults<GetEventNamesResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching event names:", error);
- return res.status(500).send({ error: "Failed to fetch event names" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getEventProperties.ts b/server/src/api/analytics/events/getEventProperties.ts
index d0a93e42d..216e6ee20 100644
--- a/server/src/api/analytics/events/getEventProperties.ts
+++ b/server/src/api/analytics/events/getEventProperties.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetEventPropertiesResponse = {
propertyKey: string;
@@ -19,19 +19,14 @@ export interface GetEventPropertiesRequest {
}>;
}
-export async function getEventProperties(req: FastifyRequest<GetEventPropertiesRequest>, res: FastifyReply) {
- const { event_name: eventName, filters } = req.query;
- const site = req.params.siteId;
+export const buildEventPropertiesQuery = (query: GetEventPropertiesRequest["Querystring"], siteId: number) => {
+ const { filters } = query;
- if (!eventName) {
- return res.status(400).send({ error: "Event name is required" });
- }
-
- const timeStatement = getTimeStatement(req.query);
+ const timeStatement = getTimeStatement(query);
- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";
+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";
- const query = `
+ return `
SELECT
kv.1 AS propertyKey, -- Access tuple elements
replaceRegexpAll(kv.2, '^"|"$', '') AS propertyValue, -- Remove surrounding quotes if they exist
@@ -49,22 +44,26 @@ export async function getEventProperties(req: FastifyRequest<GetEventPropertiesR
ORDER BY propertyKey ASC, count DESC
LIMIT 500
`;
+};
+
+export const getEventProperties = analyticsRoute<GetEventPropertiesRequest>(
+ "event properties",
+ async (req: FastifyRequest<GetEventPropertiesRequest>, res: FastifyReply) => {
+ const { event_name: eventName } = req.query;
+ const site = req.params.siteId;
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ if (!eventName) {
+ return res.status(400).send({ error: "Event name is required" });
+ }
+
+ const data = await runAnalyticsQuery<GetEventPropertiesResponse[number]>({
+ query: buildEventPropertiesQuery(req.query, Number(site)),
+ params: {
siteId: Number(site),
eventName,
},
});
- const data = await processResults<GetEventPropertiesResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching event properties:", error);
- return res.status(500).send({ error: "Failed to fetch event properties" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getEvents.ts b/server/src/api/analytics/events/getEvents.ts
index 4f057b908..6777c0731 100644
--- a/server/src/api/analytics/events/getEvents.ts
+++ b/server/src/api/analytics/events/getEvents.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { enrichWithTraits, processResults, getTimeStatement } from "../utils/utils.js";
+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, QuerySpec, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetEventsResponse = {
timestamp: string;
@@ -73,27 +73,16 @@ const EVENT_COLUMNS = `
const EVENT_TYPE_FILTER = `AND type IN ('custom_event', 'pageview', 'outbound', 'button_click', 'copy', 'form_submit', 'input_change')`;
-export async function getEvents(
- req: FastifyRequest<GetEventsRequest>,
- res: FastifyReply
-) {
- const { siteId } = req.params;
- const {
- since_timestamp,
- before_timestamp,
- page_size: pageSize = "50",
- filters,
- } = req.query;
+export const buildEventsQuery = (query: GetEventsRequest["Querystring"], siteId: number): QuerySpec => {
+ const { since_timestamp, before_timestamp, page_size: pageSize = "50", filters } = query;
const limit = parseInt(pageSize, 10);
- const filterStatement = filters
- ? getFilterStatement(filters, Number(siteId))
- : "";
+ const filterStatement = filters ? getFilterStatement(filters, siteId) : "";
- try {
- // Mode A: Poll for new events since a timestamp (Realtime polling)
- if (since_timestamp) {
- const query = `
+ // Mode A: Poll for new events since a timestamp (Realtime polling)
+ if (since_timestamp) {
+ return {
+ query: `
SELECT ${EVENT_COLUMNS}
FROM events
WHERE
@@ -103,40 +92,30 @@ export async function getEvents(
${filterStatement}
ORDER BY timestamp DESC
LIMIT 500
- `;
-
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(siteId),
- sinceTimestamp: since_timestamp,
- },
- });
-
- const events = await processResults<GetEventsResponse[number]>(result);
- const eventsWithTraits = await enrichWithTraits(events, Number(siteId));
- return res.send({ data: eventsWithTraits });
- }
+ `,
+ params: {
+ siteId,
+ sinceTimestamp: since_timestamp,
+ },
+ };
+ }
- // Mode B: Cursor-based pagination (initial load or scrolling back)
- const timeStatement =
- req.query.start_date || req.query.end_date
- ? getTimeStatement(req.query)
- : "";
+ // Mode B: Cursor-based pagination (initial load or scrolling back)
+ const timeStatement = query.start_date || query.end_date ? getTimeStatement(query) : "";
- let cursorCondition = "";
- const queryParams: Record<string, string | number> = {
- siteId: Number(siteId),
- limit: Number(limit),
- };
+ let cursorCondition = "";
+ const queryParams: Record<string, string | number> = {
+ siteId,
+ limit: Number(limit),
+ };
- if (before_timestamp) {
- cursorCondition = `AND timestamp < toDateTime64({beforeTimestamp:String}, 3)`;
- queryParams.beforeTimestamp = before_timestamp;
- }
+ if (before_timestamp) {
+ cursorCondition = `AND timestamp < toDateTime64({beforeTimestamp:String}, 3)`;
+ queryParams.beforeTimestamp = before_timestamp;
+ }
- const query = `
+ return {
+ query: `
SELECT ${EVENT_COLUMNS}
FROM events
WHERE
@@ -147,27 +126,33 @@ export async function getEvents(
${filterStatement}
ORDER BY timestamp DESC
LIMIT {limit:Int32}
- `;
+ `,
+ params: queryParams,
+ };
+};
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: queryParams,
- });
+export const getEvents = analyticsRoute<GetEventsRequest>(
+ "events",
+ async (req: FastifyRequest<GetEventsRequest>, res: FastifyReply) => {
+ const { siteId } = req.params;
+ const { since_timestamp, page_size: pageSize = "50" } = req.query;
- const events = await processResults<GetEventsResponse[number]>(result);
+ const events = await runAnalyticsQuery<GetEventsResponse[number]>(buildEventsQuery(req.query, Number(siteId)));
const eventsWithTraits = await enrichWithTraits(events, Number(siteId));
+ // Mode A: Poll for new events since a timestamp (Realtime polling)
+ if (since_timestamp) {
+ return res.send({ data: eventsWithTraits });
+ }
+
+ // Mode B: Cursor-based pagination (initial load or scrolling back)
+ const limit = parseInt(pageSize, 10);
return res.send({
data: eventsWithTraits,
cursor: {
hasMore: events.length === limit,
- oldestTimestamp:
- events.length > 0 ? events[events.length - 1].timestamp : null,
+ oldestTimestamp: events.length > 0 ? events[events.length - 1].timestamp : null,
},
});
- } catch (error) {
- console.error("Error fetching events:", error);
- return res.status(500).send({ error: "Failed to fetch events" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getOutboundLinks.ts b/server/src/api/analytics/events/getOutboundLinks.ts
index 2bc4398c9..2c83a1344 100644
--- a/server/src/api/analytics/events/getOutboundLinks.ts
+++ b/server/src/api/analytics/events/getOutboundLinks.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetOutboundLinksResponse = {
url: string;
@@ -17,14 +17,13 @@ export interface GetOutboundLinksRequest {
Querystring: FilterParams<{}>;
}
-export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksRequest>, res: FastifyReply) {
- const { filters } = req.query;
- const site = req.params.siteId;
+export const buildOutboundLinksQuery = (query: GetOutboundLinksRequest["Querystring"], siteId: number) => {
+ const { filters } = query;
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";
- const query = `
+ return `
SELECT
JSONExtractString(toString(props), 'url') AS url,
COUNT(*) AS count,
@@ -39,21 +38,18 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksReque
ORDER BY count DESC
LIMIT 1000
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+export const getOutboundLinks = analyticsRoute<GetOutboundLinksRequest>(
+ "outbound links",
+ async (req: FastifyRequest<GetOutboundLinksRequest>, res: FastifyReply) => {
+ const site = req.params.siteId;
+
+ const data = await runAnalyticsQuery<GetOutboundLinksResponse[number]>({
+ query: buildOutboundLinksQuery(req.query, Number(site)),
+ params: { siteId: Number(site) },
});
- const data = await processResults<GetOutboundLinksResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching outbound links:", error);
- return res.status(500).send({ error: "Failed to fetch outbound links" });
}
-}
+);
diff --git a/server/src/api/analytics/events/getSiteEventCount.ts b/server/src/api/analytics/events/getSiteEventCount.ts
index f661b116b..affdd2c6b 100644
--- a/server/src/api/analytics/events/getSiteEventCount.ts
+++ b/server/src/api/analytics/events/getSiteEventCount.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { TimeBucket } from "../types.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";
+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetSiteEventCountResponse = {
time: string;
@@ -28,22 +28,13 @@ interface GetSiteEventCountRequest {
}>;
}
-export async function getSiteEventCount(
- req: FastifyRequest<GetSiteEventCountRequest>,
- res: FastifyReply
-) {
- const site = req.params.siteId;
- const { bucket = "day" } = req.query;
- const timeZone = req.query.time_zone || "UTC";
+export const buildSiteEventCountQuery = (query: GetSiteEventCountRequest["Querystring"], siteId: number) => {
+ const { bucket = "day" } = query;
- if (!TimeBucketToFn[bucket]) {
- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
- }
-
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(req.query.filters, Number(site), timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
- const query = `
+ return `
SELECT
toDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, {timeZone:String}))) AS time,
countIf(type = 'pageview') as pageview_count,
@@ -65,22 +56,27 @@ export async function getSiteEventCount(
GROUP BY time
ORDER BY time
`;
+};
+
+export const getSiteEventCount = analyticsRoute<GetSiteEventCountRequest>(
+ "site event count",
+ async (req: FastifyRequest<GetSiteEventCountRequest>, res: FastifyReply) => {
+ const site = req.params.siteId;
+ const { bucket = "day" } = req.query;
+ const timeZone = req.query.time_zone || "UTC";
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ if (!TimeBucketToFn[bucket]) {
+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
+ }
+
+ const data = await runAnalyticsQuery<GetSiteEventCountResponse[number]>({
+ query: buildSiteEventCountQuery(req.query, Number(site)),
+ params: {
siteId: Number(site),
timeZone,
},
});
- const data = await processResults<GetSiteEventCountResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching site event count:", error);
- return res.status(500).send({ error: "Failed to fetch site event count" });
}
-}
+);
diff --git a/server/src/api/analytics/funnels/getFunnel.ts b/server/src/api/analytics/funnels/getFunnel.ts
index bf8f042d9..976de1614 100644
--- a/server/src/api/analytics/funnels/getFunnel.ts
+++ b/server/src/api/analytics/funnels/getFunnel.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { AnalyticsQueryError, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { buildFunnelStepCondition, FunnelStep } from "./funnelSteps.js";
type Funnel = {
@@ -18,33 +18,15 @@ type FunnelResponse = {
dropoff_rate: number;
};
-export async function getFunnel(
- request: FastifyRequest<{
- Body: Funnel;
- Params: {
- siteId: string;
- };
- Querystring: FilterParams<{}>;
- }>,
- reply: FastifyReply
-) {
- const { steps } = request.body;
- const { siteId } = request.params;
-
- // Validate request
- if (!steps || steps.length < 2) {
- return reply.status(400).send({ error: "At least 2 steps are required for a funnel" });
- }
+export const buildFunnelQuery = (query: FilterParams<{}>, siteId: number, steps: FunnelStep[]) => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
- try {
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(request.query.filters, Number(siteId), timeStatement);
+ // Build conditional statements for each step
+ const stepConditions = steps.map(step => buildFunnelStepCondition(step));
- // Build conditional statements for each step
- const stepConditions = steps.map(step => buildFunnelStepCondition(step));
-
- // Build the funnel query - session-based tracking
- const query = `
+ // Build the funnel query - session-based tracking
+ return `
WITH
-- Get all session actions in the time period
SessionActions AS (
@@ -107,14 +89,14 @@ export async function getFunnel(
)
.join("\nUNION ALL\n")}
)
-
+
-- Final results with calculated conversion and dropoff rates
SELECT
s1.step_number,
s1.step_name,
s1.visitors as visitors,
round(s1.visitors * 100.0 / first_step.visitors, 2) as conversion_rate,
- CASE
+ CASE
WHEN s1.step_number = 1 THEN 0
ELSE round((1 - (s1.visitors / prev_step.visitors)) * 100.0, 2)
END as dropoff_rate
@@ -127,22 +109,45 @@ export async function getFunnel(
) as prev_step ON s1.step_number = prev_step.next_step_number
ORDER BY s1.step_number
`;
+};
+
+export async function getFunnel(
+ request: FastifyRequest<{
+ Body: Funnel;
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{}>;
+ }>,
+ reply: FastifyReply
+) {
+ const { steps } = request.body;
+ const { siteId } = request.params;
+
+ // Validate request
+ if (!steps || steps.length < 2) {
+ return reply.status(400).send({ error: "At least 2 steps are required for a funnel" });
+ }
- // Execute the query
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ try {
+ const data = await runAnalyticsQuery<FunnelResponse>({
+ query: buildFunnelQuery(request.query, Number(siteId), steps),
+ params: {
siteId: Number(siteId),
stepNumber: steps.length,
},
});
- // Process the results
- const data = await processResults<FunnelResponse>(result);
return reply.send({ data });
} catch (error) {
- console.error("Error executing funnel query:", error);
+ if (error instanceof AnalyticsQueryError) {
+ console.error("Error executing funnel query:", error.original);
+ for (const query of error.queries) {
+ console.error("Failed query:", query);
+ }
+ } else {
+ console.error("Error executing funnel query:", error);
+ }
return reply.status(500).send({ error: "Failed to execute funnel analysis" });
}
}
diff --git a/server/src/api/analytics/funnels/getFunnelStepSessions.ts b/server/src/api/analytics/funnels/getFunnelStepSessions.ts
index c51800f78..6272f5387 100644
--- a/server/src/api/analytics/funnels/getFunnelStepSessions.ts
+++ b/server/src/api/analytics/funnels/getFunnelStepSessions.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";
import { GetSessionsResponse } from "../sessions/getSessions.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { buildFunnelStepCondition, FunnelStep } from "./funnelSteps.js";
type Funnel = {
@@ -23,47 +23,28 @@ export interface GetFunnelStepSessionsRequest {
}>;
}
-export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSessionsRequest>, res: FastifyReply) {
- const { steps } = req.body;
- const { stepNumber: stepNumberStr, siteId } = req.params;
- const { mode, page, limit } = req.query;
-
- const stepNumber = parseInt(stepNumberStr, 10);
-
- // Validate request
- if (!steps || steps.length < 2) {
- return res.status(400).send({ error: "At least 2 steps are required for a funnel" });
- }
-
- if (isNaN(stepNumber) || stepNumber < 1 || stepNumber > steps.length) {
- return res.status(400).send({ error: "Invalid step number" });
- }
-
- if (mode !== "reached" && mode !== "dropped") {
- return res.status(400).send({ error: "Mode must be 'reached' or 'dropped'" });
- }
-
- // For final step in "dropped" mode, return empty results (no drop-off possible)
- if (mode === "dropped" && stepNumber === steps.length) {
- return res.send({ data: [] });
- }
-
- try {
- const timeStatement = getTimeStatement(req.query);
-
- // Applied once, in the SessionActions CTE against raw events (same as the
- // funnel analyze endpoint), where every filter parameter's column exists.
- // Re-applying it to the aggregated outer query breaks on parameters the
- // aggregate doesn't project (utm_*, pathname, timezone → unknown identifier).
- const filterStatement = getFilterStatement(req.query.filters, Number(siteId), timeStatement);
-
- // Build conditional statements for each step we need
- const stepsToCheck = mode === "reached" ? stepNumber : stepNumber + 1;
- const stepConditions = steps.slice(0, stepsToCheck).map(step => buildFunnelStepCondition(step));
-
- // Build CTEs for each funnel step to identify qualifying sessions
- const stepCTEs = [
- `
+export const buildFunnelStepSessionsQuery = (
+ query: GetFunnelStepSessionsRequest["Querystring"],
+ siteId: number,
+ steps: FunnelStep[],
+ stepNumber: number
+) => {
+ const { mode } = query;
+ const timeStatement = getTimeStatement(query);
+
+ // Applied once, in the SessionActions CTE against raw events (same as the
+ // funnel analyze endpoint), where every filter parameter's column exists.
+ // Re-applying it to the aggregated outer query breaks on parameters the
+ // aggregate doesn't project (utm_*, pathname, timezone → unknown identifier).
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
+
+ // Build conditional statements for each step we need
+ const stepsToCheck = mode === "reached" ? stepNumber : stepNumber + 1;
+ const stepConditions = steps.slice(0, stepsToCheck).map(step => buildFunnelStepCondition(step));
+
+ // Build CTEs for each funnel step to identify qualifying sessions
+ const stepCTEs = [
+ `
SessionActions AS (
SELECT
session_id,
@@ -89,11 +70,11 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSes
WHERE ${stepConditions[0]}
GROUP BY session_id
)`,
- ];
+ ];
- // Add CTEs for steps 2 through stepNumber (and +1 for dropped mode)
- for (let i = 1; i < stepsToCheck; i++) {
- stepCTEs.push(`
+ // Add CTEs for steps 2 through stepNumber (and +1 for dropped mode)
+ for (let i = 1; i < stepsToCheck; i++) {
+ stepCTEs.push(`
Step${i + 1} AS (
SELECT DISTINCT
s${i}.session_id,
@@ -105,20 +86,20 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSes
AND ${stepConditions[i]}
GROUP BY s${i}.session_id
)`);
- }
+ }
- // Determine which sessions to retrieve
- let targetSessionsCTE = "";
- if (mode === "reached") {
- // Sessions that completed step N
- targetSessionsCTE = `
+ // Determine which sessions to retrieve
+ let targetSessionsCTE = "";
+ if (mode === "reached") {
+ // Sessions that completed step N
+ targetSessionsCTE = `
TargetSessions AS (
SELECT session_id
FROM Step${stepNumber}
)`;
- } else {
- // Sessions that completed step N but NOT step N+1
- targetSessionsCTE = `
+ } else {
+ // Sessions that completed step N but NOT step N+1
+ targetSessionsCTE = `
TargetSessions AS (
SELECT session_id
FROM Step${stepNumber}
@@ -127,10 +108,10 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSes
FROM Step${stepNumber + 1}
)
)`;
- }
+ }
- // Build main query to aggregate session data
- const query = `
+ // Build main query to aggregate session data
+ return `
WITH
${stepCTEs.join(",\n")}
,
@@ -189,22 +170,45 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSes
FROM AggregatedSessions
LIMIT {limit:Int32} OFFSET {offset:Int32}
`;
+};
+
+export const getFunnelStepSessions = analyticsRoute<GetFunnelStepSessionsRequest>(
+ "funnel step sessions",
+ async (req: FastifyRequest<GetFunnelStepSessionsRequest>, res: FastifyReply) => {
+ const { steps } = req.body;
+ const { stepNumber: stepNumberStr, siteId } = req.params;
+ const { mode, page, limit } = req.query;
+
+ const stepNumber = parseInt(stepNumberStr, 10);
+
+ // Validate request
+ if (!steps || steps.length < 2) {
+ return res.status(400).send({ error: "At least 2 steps are required for a funnel" });
+ }
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ if (isNaN(stepNumber) || stepNumber < 1 || stepNumber > steps.length) {
+ return res.status(400).send({ error: "Invalid step number" });
+ }
+
+ if (mode !== "reached" && mode !== "dropped") {
+ return res.status(400).send({ error: "Mode must be 'reached' or 'dropped'" });
+ }
+
+ // For final step in "dropped" mode, return empty results (no drop-off possible)
+ if (mode === "dropped" && stepNumber === steps.length) {
+ return res.send({ data: [] });
+ }
+
+ const data = await runAnalyticsQuery<Omit<GetSessionsResponse[number], "traits">>({
+ query: buildFunnelStepSessionsQuery(req.query, Number(siteId), steps, stepNumber),
+ params: {
siteId: Number(siteId),
limit: limit || 25,
offset: ((page || 1) - 1) * (limit || 25),
},
});
- const data = await processResults<Omit<GetSessionsResponse[number], "traits">>(result);
const dataWithTraits = await enrichWithTraits(data, Number(siteId));
return res.send({ data: dataWithTraits });
- } catch (error) {
- console.error("Error fetching funnel step sessions:", error);
- return res.status(500).send({ error: "Failed to fetch funnel step sessions" });
}
-}
+);
diff --git a/server/src/api/analytics/getErrorBucketed.ts b/server/src/api/analytics/getErrorBucketed.ts
index 4ed7029b5..aba98c669 100644
--- a/server/src/api/analytics/getErrorBucketed.ts
+++ b/server/src/api/analytics/getErrorBucketed.ts
@@ -1,11 +1,11 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { validateTimeStatementFillParams } from "./utils/query-validation.js";
-import { getTimeStatement, TimeBucketToFn, bucketIntervalMap, processResults } from "./utils/utils.js";
+import { getTimeStatement, TimeBucketToFn, bucketIntervalMap } from "./utils/utils.js";
import { TimeBucket } from "./types.js";
import { getFilterStatement } from "./utils/getFilterStatement.js";
+import { AnalyticsQueryError, runAnalyticsQuery } from "./utils/analyticsQuery.js";
function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);
@@ -58,21 +58,13 @@ export type GetErrorBucketedResponse = {
error_count: number;
}[];
-export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedRequest>, res: FastifyReply) {
- const site = req.params.siteId;
- const { bucket, errorMessage } = req.query;
-
- if (!errorMessage) {
- return res.status(400).send({ error: "errorMessage parameter is required" });
- }
-
- const numericSiteId = Number(site);
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(req.query.filters, numericSiteId, timeStatement);
- const timeStatementFill = getTimeStatementFill(req.query, bucket);
+export const buildErrorBucketedQuery = (query: GetErrorBucketedRequest["Querystring"], siteId: number) => {
+ const { bucket } = query;
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
+ const timeStatementFill = getTimeStatementFill(query, bucket);
- try {
- const query = `
+ return `
SELECT
${TimeBucketToFn[bucket]}(toTimeZone(timestamp, {timeZone:String})) AS time,
COUNT(*) AS error_count
@@ -87,25 +79,34 @@ export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedReque
ORDER BY time
${timeStatementFill}
`;
+};
+
+export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedRequest>, res: FastifyReply) {
+ const site = req.params.siteId;
+ const { errorMessage } = req.query;
+
+ if (!errorMessage) {
+ return res.status(400).send({ error: "errorMessage parameter is required" });
+ }
+
+ const numericSiteId = Number(site);
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ try {
+ const data = await runAnalyticsQuery<GetErrorBucketedResponse[number]>({
+ query: buildErrorBucketedQuery(req.query, numericSiteId),
+ params: {
siteId: numericSiteId,
errorMessage: errorMessage,
timeZone: req.query.time_zone || "UTC",
},
});
- const data = await processResults<GetErrorBucketedResponse>(result);
-
return res.send({
success: true,
data: data,
});
} catch (error) {
- console.error("Error getting error bucketed data:", error);
+ console.error("Error getting error bucketed data:", error instanceof AnalyticsQueryError ? error.original : error);
return res.status(500).send({
success: false,
error: "Failed to get error data",
diff --git a/server/src/api/analytics/getErrorEvents.ts b/server/src/api/analytics/getErrorEvents.ts
index 548f3548b..7dd923c09 100644
--- a/server/src/api/analytics/getErrorEvents.ts
+++ b/server/src/api/analytics/getErrorEvents.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "./utils/utils.js";
+import { getTimeStatement } from "./utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "./utils/getFilterStatement.js";
+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";
interface GetErrorEventsRequest {
Params: {
@@ -46,31 +46,18 @@ type ErrorEventsPaginatedResponse = {
totalCount: number;
};
-const getErrorEventsQuery = (request: FastifyRequest<GetErrorEventsRequest>, isCountQuery: boolean = false) => {
- const { filters, limit, page } = request.query;
+export const buildErrorEventsQuery = (
+ query: GetErrorEventsRequest["Querystring"],
+ siteId: number,
+ isCountQuery: boolean = false
+) => {
+ const { filters } = query;
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
- let validatedLimit: number | null = null;
- if (!isCountQuery && limit !== undefined) {
- const parsedLimit = parseInt(String(limit), 10);
- if (!isNaN(parsedLimit) && parsedLimit > 0) {
- validatedLimit = parsedLimit;
- }
- }
// Default to 20 for error events
- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 20";
-
- let validatedOffset: number | null = null;
- if (!isCountQuery && page !== undefined) {
- const parsedPage = parseInt(String(page), 10);
- if (!isNaN(parsedPage) && parsedPage >= 1) {
- const pageOffset = (parsedPage - 1) * (validatedLimit || 20);
- validatedOffset = pageOffset;
- }
- }
- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 20, isCountQuery);
if (isCountQuery) {
return `
@@ -131,52 +118,28 @@ const getErrorEventsQuery = (request: FastifyRequest<GetErrorEventsRequest>, isC
`;
};
-export async function getErrorEvents(req: FastifyRequest<GetErrorEventsRequest>, res: FastifyReply) {
- const site = req.params.siteId;
- const { errorMessage, page } = req.query;
-
- if (!errorMessage) {
- return res.status(400).send({ error: "errorMessage parameter is required" });
- }
-
- const isPaginatedRequest = page !== undefined;
+export const getErrorEvents = analyticsRoute<GetErrorEventsRequest>(
+ "error events",
+ async (req: FastifyRequest<GetErrorEventsRequest>, res: FastifyReply) => {
+ const { errorMessage } = req.query;
- const dataQuery = getErrorEventsQuery(req, false);
+ if (!errorMessage) {
+ return res.status(400).send({ error: "errorMessage parameter is required" });
+ }
- try {
- const dataResult = await clickhouse.query({
- query: dataQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- errorMessage: errorMessage,
- },
- });
- const items = await processResults<ErrorEvent>(dataResult);
+ const siteId = Number(req.params.siteId);
+ const params = { siteId, errorMessage };
+ const dataSpec = { query: buildErrorEventsQuery(req.query, siteId, false), params };
- if (isPaginatedRequest) {
- const countQuery = getErrorEventsQuery(req, true);
- const countResult = await clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- errorMessage: errorMessage,
- },
+ if (req.query.page !== undefined) {
+ const result = await runPaginatedQuery<ErrorEvent>(dataSpec, {
+ query: buildErrorEventsQuery(req.query, siteId, true),
+ params,
});
- const countData = await processResults<{ totalCount: number }>(countResult);
- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;
- return res.send({ data: { data: items, totalCount } });
- } else {
- return res.send({ data: items });
+ return res.send({ data: result });
}
- } catch (error) {
- console.error(`Error fetching error events:`, error);
- console.error("Failed dataQuery:", dataQuery);
- if (isPaginatedRequest) {
- const countQuery = getErrorEventsQuery(req, true);
- console.error("Failed countQuery:", countQuery);
- }
- return res.status(500).send({ error: `Failed to fetch error events` });
+
+ const items = await runAnalyticsQuery<ErrorEvent>(dataSpec);
+ return res.send({ data: items });
}
-}
+);
diff --git a/server/src/api/analytics/getErrorNames.ts b/server/src/api/analytics/getErrorNames.ts
index 0426006a2..bff0056fb 100644
--- a/server/src/api/analytics/getErrorNames.ts
+++ b/server/src/api/analytics/getErrorNames.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "./utils/utils.js";
+import { getTimeStatement } from "./utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "./utils/getFilterStatement.js";
+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";
interface GetErrorNamesRequest {
Params: {
@@ -29,31 +29,18 @@ type ErrorNamesPaginatedResponse = {
totalCount: number;
};
-const getErrorNamesQuery = (request: FastifyRequest<GetErrorNamesRequest>, isCountQuery: boolean = false) => {
- const { filters, limit, page } = request.query;
+export const buildErrorNamesQuery = (
+ query: GetErrorNamesRequest["Querystring"],
+ siteId: number,
+ isCountQuery: boolean = false
+) => {
+ const { filters } = query;
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
- let validatedLimit: number | null = null;
- if (!isCountQuery && limit !== undefined) {
- const parsedLimit = parseInt(String(limit), 10);
- if (!isNaN(parsedLimit) && parsedLimit > 0) {
- validatedLimit = parsedLimit;
- }
- }
// Default to 10 for non-paginated use
- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 10";
-
- let validatedOffset: number | null = null;
- if (!isCountQuery && page !== undefined) {
- const parsedPage = parseInt(String(page), 10);
- if (!isNaN(parsedPage) && parsedPage >= 1) {
- const pageOffset = (parsedPage - 1) * (validatedLimit || 10);
- validatedOffset = pageOffset;
- }
- }
- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 10, isCountQuery);
// For errors, we want to count total occurrences and unique sessions affected
// Group by error message instead of error name
@@ -103,47 +90,23 @@ const getErrorNamesQuery = (request: FastifyRequest<GetErrorNamesRequest>, isCou
`;
};
-export async function getErrorNames(req: FastifyRequest<GetErrorNamesRequest>, res: FastifyReply) {
- const site = req.params.siteId;
- const { page } = req.query;
-
- const isPaginatedRequest = page !== undefined; // True if page is present
+export const getErrorNames = analyticsRoute<GetErrorNamesRequest>(
+ "error names",
+ async (req: FastifyRequest<GetErrorNamesRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const params = { siteId };
+ const dataSpec = { query: buildErrorNamesQuery(req.query, siteId, false), params };
- const dataQuery = getErrorNamesQuery(req, false);
-
- try {
- const dataResult = await clickhouse.query({
- query: dataQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- });
- const items = await processResults<ErrorNameItem>(dataResult);
-
- if (isPaginatedRequest) {
- const countQuery = getErrorNamesQuery(req, true);
- const countResult = await clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ if (req.query.page !== undefined) {
+ const result = await runPaginatedQuery<ErrorNameItem>(dataSpec, {
+ query: buildErrorNamesQuery(req.query, siteId, true),
+ params,
});
- const countData = await processResults<{ totalCount: number }>(countResult);
- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;
- return res.send({ data: { data: items, totalCount } });
- } else {
- // For non-paginated (StandardSection default) use, return the simpler structure
- return res.send({ data: items });
- }
- } catch (error) {
- console.error(`Error fetching error names:`, error);
- console.error("Failed dataQuery:", dataQuery);
- if (isPaginatedRequest) {
- const countQuery = getErrorNamesQuery(req, true);
- console.error("Failed countQuery:", countQuery);
+ return res.send({ data: result });
}
- return res.status(500).send({ error: `Failed to fetch error names` });
+
+ // For non-paginated (StandardSection default) use, return the simpler structure
+ const items = await runAnalyticsQuery<ErrorNameItem>(dataSpec);
+ return res.send({ data: items });
}
-}
+);
diff --git a/server/src/api/analytics/getJourneys.ts b/server/src/api/analytics/getJourneys.ts
index 919ee85f3..af5bff118 100644
--- a/server/src/api/analytics/getJourneys.ts
+++ b/server/src/api/analytics/getJourneys.ts
@@ -2,9 +2,9 @@ import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
import { z } from "zod";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { getFilterStatement } from "./utils/getFilterStatement.js";
import { getTimeStatement, patternToRegex } from "./utils/utils.js";
+import { AnalyticsQueryError, runAnalyticsQuery } from "./utils/analyticsQuery.js";
// stepFilters arrives as a JSON object mapping a (numeric) step index to a path
// pattern. Both the keys and values are attacker-controlled, so validate the
@@ -14,71 +14,48 @@ const stepFiltersSchema = z.record(
z.string().max(2048)
);
-export const getJourneys = async (
- request: FastifyRequest<{
- Params: { siteId: string };
- Querystring: FilterParams<{
- steps?: string;
- limit?: string;
- stepFilters?: string;
- }>;
- }>,
- reply: FastifyReply
-) => {
- try {
- const { siteId } = request.params;
- const { steps = "3", limit = "100", filters, stepFilters } = request.query;
-
- const maxSteps = parseInt(steps, 10);
- const journeyLimit = parseInt(limit, 10);
-
- if (isNaN(maxSteps) || maxSteps < 2 || maxSteps > 10) {
- return reply.status(400).send({
- error: "Steps parameter must be a number between 2 and 10",
- });
- }
+interface GetJourneysRequest {
+ Params: { siteId: string };
+ Querystring: FilterParams<{
+ steps?: string;
+ limit?: string;
+ stepFilters?: string;
+ }>;
+}
- if (isNaN(journeyLimit) || journeyLimit < 1 || journeyLimit > 500) {
- return reply.status(400).send({
- error: "Limit parameter must be a number between 1 and 500",
- });
- }
+type JourneyRow = {
+ journey: string[];
+ sessions_count: number;
+ percentage: number;
+};
- // Time conditions using getTimeStatement
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(filters, Number(siteId), timeStatement);
+export const buildJourneysQuery = (
+ query: GetJourneysRequest["Querystring"],
+ siteId: number,
+ parsedStepFilters: Record<string, string>
+) => {
+ // Time conditions using getTimeStatement
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
- // Parse and validate step filters
- let parsedStepFilters: Record<string, string> = {};
- if (stepFilters) {
- try {
- parsedStepFilters = stepFiltersSchema.parse(JSON.parse(stepFilters));
- } catch (error) {
- return reply.status(400).send({
- error: "Invalid stepFilters format",
- });
+ // Build step filter conditions for the HAVING clause
+ // Supports wildcard patterns: * matches single segment, ** matches multiple segments
+ const stepFilterConditions = Object.entries(parsedStepFilters)
+ .map(([step, path]) => {
+ const stepIndex = parseInt(step, 10) + 1; // ClickHouse arrays are 1-indexed
+ if (path.includes("*")) {
+ // Use regex matching for wildcard patterns. SqlString.escape correctly
+ // escapes the regex literal for ClickHouse (handles both ' and \).
+ const regex = patternToRegex(path);
+ return `match(journey[${stepIndex}], ${SqlString.escape(regex)})`;
}
- }
-
- // Build step filter conditions for the HAVING clause
- // Supports wildcard patterns: * matches single segment, ** matches multiple segments
- const stepFilterConditions = Object.entries(parsedStepFilters)
- .map(([step, path]) => {
- const stepIndex = parseInt(step, 10) + 1; // ClickHouse arrays are 1-indexed
- if (path.includes("*")) {
- // Use regex matching for wildcard patterns. SqlString.escape correctly
- // escapes the regex literal for ClickHouse (handles both ' and \).
- const regex = patternToRegex(path);
- return `match(journey[${stepIndex}], ${SqlString.escape(regex)})`;
- }
- // Use exact match for non-wildcard patterns (more efficient)
- return `journey[${stepIndex}] = ${SqlString.escape(path)}`;
- })
- .join(" AND ");
+ // Use exact match for non-wildcard patterns (more efficient)
+ return `journey[${stepIndex}] = ${SqlString.escape(path)}`;
+ })
+ .join(" AND ");
- // Query to find sequences of events (journeys) for each user
- const result = await clickhouse.query({
- query: `
+ // Query to find sequences of events (journeys) for each user
+ return `
WITH user_paths AS (
SELECT
session_id,
@@ -122,25 +99,59 @@ export const getJourneys = async (
${filterStatement || ""}
) AS percentage
FROM journey_segments
- `,
- query_params: {
+ `;
+};
+
+export const getJourneys = async (request: FastifyRequest<GetJourneysRequest>, reply: FastifyReply) => {
+ try {
+ const { siteId } = request.params;
+ const { steps = "3", limit = "100", stepFilters } = request.query;
+
+ const maxSteps = parseInt(steps, 10);
+ const journeyLimit = parseInt(limit, 10);
+
+ if (isNaN(maxSteps) || maxSteps < 2 || maxSteps > 10) {
+ return reply.status(400).send({
+ error: "Steps parameter must be a number between 2 and 10",
+ });
+ }
+
+ if (isNaN(journeyLimit) || journeyLimit < 1 || journeyLimit > 500) {
+ return reply.status(400).send({
+ error: "Limit parameter must be a number between 1 and 500",
+ });
+ }
+
+ // Parse and validate step filters
+ let parsedStepFilters: Record<string, string> = {};
+ if (stepFilters) {
+ try {
+ parsedStepFilters = stepFiltersSchema.parse(JSON.parse(stepFilters));
+ } catch (error) {
+ return reply.status(400).send({
+ error: "Invalid stepFilters format",
+ });
+ }
+ }
+
+ const data = await runAnalyticsQuery<JourneyRow>({
+ query: buildJourneysQuery(request.query, Number(siteId), parsedStepFilters),
+ params: {
siteId: parseInt(siteId, 10),
maxSteps: maxSteps,
journeyLimit: journeyLimit,
},
});
- const data = await result.json();
-
return reply.send({
- journeys: data.data.map((item: any) => ({
+ journeys: data.map(item => ({
path: item.journey,
count: Number(item.sessions_count),
percentage: Number(item.percentage),
})),
});
} catch (error) {
- console.error("Error getting journeys:", error);
+ console.error("Error getting journeys:", error instanceof AnalyticsQueryError ? error.original : error);
return reply.status(500).send({ error: "Failed to get journeys" });
}
};
diff --git a/server/src/api/analytics/getLiveUsercount.ts b/server/src/api/analytics/getLiveUsercount.ts
index 37a11048b..8da81630d 100644
--- a/server/src/api/analytics/getLiveUsercount.ts
+++ b/server/src/api/analytics/getLiveUsercount.ts
@@ -1,27 +1,28 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { processResults } from "./utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
-export const getLiveUsercount = async (
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: { minutes: number };
- }>,
- res: FastifyReply
-) => {
- const { siteId } = req.params;
- const { minutes } = req.query;
+interface GetLiveUsercountRequest {
+ Params: { siteId: string };
+ Querystring: { minutes: number };
+}
- const query = await clickhouse.query({
- query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval {minutes:Int32} minute AND site_id = {siteId:Int32}`,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(siteId),
- minutes: Number(minutes || 5),
- },
- });
+export const buildLiveUsercountQuery = () =>
+ `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval {minutes:Int32} minute AND site_id = {siteId:Int32}`;
- const result = await processResults<{ count: number }>(query);
+export const getLiveUsercount = analyticsRoute<GetLiveUsercountRequest>(
+ "live user count",
+ async (req: FastifyRequest<GetLiveUsercountRequest>, res: FastifyReply) => {
+ const { siteId } = req.params;
+ const { minutes } = req.query;
- return res.send({ count: result[0].count });
-};
+ const result = await runAnalyticsQuery<{ count: number }>({
+ query: buildLiveUsercountQuery(),
+ params: {
+ siteId: Number(siteId),
+ minutes: Number(minutes || 5),
+ },
+ });
+
+ return res.send({ count: result[0].count });
+ }
+);
diff --git a/server/src/api/analytics/getMetric.test.ts b/server/src/api/analytics/getMetric.test.ts
new file mode 100644
index 000000000..bbef713d0
--- /dev/null
+++ b/server/src/api/analytics/getMetric.test.ts
@@ -0,0 +1,132 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../../db/clickhouse/clickhouse.js", () => ({
+ clickhouse: { query: vi.fn() },
+}));
+vi.mock("../../db/postgres/postgres.js", () => ({
+ db: {},
+}));
+
+import { buildMetricQuery } from "./getMetric.js";
+import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";
+import { FilterParameter } from "./types.js";
+
+const baseQuery = (overrides: Partial<Record<string, unknown>> = {}) =>
+ ({
+ start_date: "",
+ end_date: "",
+ time_zone: "",
+ filters: "",
+ parameter: "browser" as FilterParameter,
+ ...overrides,
+ }) as Parameters<typeof buildMetricQuery>[0];
+
+const SITE_ID = 1;
+
+describe("buildMetricQuery", () => {
+ describe("default (session-attributed) parameters", () => {
+ it("aggregates the parameter per session with argMin", () => {
+ const sql = buildMetricQuery(baseQuery(), SITE_ID);
+ expect(sql).toContain("argMin(browser, e.timestamp) as value");
+ expect(sql).toContain("GROUP BY e.session_id");
+ expect(sql).toContain("LIMIT 100");
+ });
+
+ it("uses the session-channel aggregation for channel", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "channel" }), SITE_ID);
+ expect(sql).toContain(`${SESSION_CHANNEL_AGG} as value`);
+ expect(sql).not.toContain("argMin(channel, e.timestamp)");
+ });
+
+ it("counts distinct values in the count query without pagination", () => {
+ const sql = buildMetricQuery(baseQuery({ limit: 25, page: 3 }), SITE_ID, true);
+ expect(sql).toContain("COUNT(DISTINCT value) as totalCount");
+ expect(sql).not.toContain("LIMIT");
+ expect(sql).not.toContain("OFFSET");
+ });
+ });
+
+ describe("event_name", () => {
+ it("restricts to custom events", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "event_name" }), SITE_ID);
+ expect(sql).toContain("event_name as value");
+ expect(sql).toContain("AND type = 'custom_event'");
+ });
+
+ it("counts distinct event names", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "event_name" }), SITE_ID, true);
+ expect(sql).toContain("COUNT(DISTINCT event_name) as totalCount");
+ });
+ });
+
+ describe("page_title", () => {
+ it("computes bounce rate from single-pageview sessions", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "page_title" }), SITE_ID);
+ expect(sql).toContain("SessionPageCounts");
+ expect(sql).toContain("pageviews_in_session = 1");
+ expect(sql).toContain("as bounce_rate");
+ });
+
+ it("wraps the core logic for the count query", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "page_title" }), SITE_ID, true);
+ expect(sql).toContain("SELECT COUNT(*) as totalCount FROM (");
+ });
+ });
+
+ describe("entry_page / exit_page", () => {
+ it("orders sessions ascending for entry pages", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);
+ expect(sql).toContain("ORDER BY timestamp ASC) as row_num");
+ expect(sql).toContain("WHERE row_num = 1");
+ });
+
+ it("orders sessions descending for exit pages", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "exit_page" }), SITE_ID);
+ expect(sql).toContain("ORDER BY timestamp DESC) as row_num");
+ });
+
+ it("caps time-on-page at 30 minutes", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);
+ expect(sql).toContain("time_diff_seconds > 1800, 1800");
+ });
+ });
+
+ describe("pathname", () => {
+ it("aggregates per-path stats with hostname", () => {
+ const sql = buildMetricQuery(baseQuery({ parameter: "pathname" }), SITE_ID);
+ expect(sql).toContain("anyHeavy(hostname) as top_hostname");
+ expect(sql).toContain("GROUP BY pathname");
+ });
+ });
+
+ describe("pagination", () => {
+ it("applies explicit limit and page offsets", () => {
+ const sql = buildMetricQuery(baseQuery({ limit: 7, page: 3 }), SITE_ID);
+ expect(sql).toContain("LIMIT 7");
+ expect(sql).toContain("OFFSET 14");
+ });
+
+ it("omits OFFSET on the first page", () => {
+ const sql = buildMetricQuery(baseQuery({ limit: 7, page: 1 }), SITE_ID);
+ expect(sql).toContain("LIMIT 7");
+ expect(sql).not.toContain("OFFSET");
+ });
+ });
+
+ describe("time and filter statements", () => {
+ it("constrains by the requested date range", () => {
+ const sql = buildMetricQuery(
+ baseQuery({ start_date: "2024-01-01", end_date: "2024-01-31", time_zone: "UTC" }),
+ SITE_ID
+ );
+ expect(sql).toContain("AND timestamp >=");
+ expect(sql).toContain("'2024-01-01'");
+ });
+
+ it("threads filters into the WHERE clause", () => {
+ const filters = JSON.stringify([{ parameter: "country", type: "equals", value: ["US"] }]);
+ const sql = buildMetricQuery(baseQuery({ filters }), SITE_ID);
+ expect(sql).toContain("'US'");
+ });
+ });
+});
diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.ts
index 6f11db71c..be7ec83f5 100644
--- a/server/src/api/analytics/getMetric.ts
+++ b/server/src/api/analytics/getMetric.ts
@@ -1,10 +1,10 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { FilterParameter } from "./types.js";
-import { getTimeStatement, processResults } from "./utils/utils.js";
+import { getTimeStatement } from "./utils/utils.js";
import { getFilterStatement, getSqlParam } from "./utils/getFilterStatement.js";
import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";
import { FilterParams } from "@rybbit/shared";
+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "./utils/analyticsQuery.js";
interface GetMetricRequest {
Params: {
@@ -53,32 +53,18 @@ type GetMetricPaginatedResponse = {
totalCount: number;
};
-const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boolean = false) => {
- const { filters, parameter, limit, page } = request.query;
- const site = request.params.siteId;
+export const buildMetricQuery = (
+ query: GetMetricRequest["Querystring"],
+ siteId: number,
+ isCountQuery: boolean = false
+) => {
+ const { filters, parameter } = query;
- const timeStatement = getTimeStatement(request.query);
+ const timeStatement = getTimeStatement(query);
- const filterStatement = getFilterStatement(filters, Number(site), timeStatement);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
- let validatedLimit: number | null = null;
- if (!isCountQuery && limit !== undefined) {
- const parsedLimit = parseInt(String(limit), 10);
- if (!isNaN(parsedLimit) && parsedLimit > 0) {
- validatedLimit = parsedLimit;
- }
- }
- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 100";
-
- let validatedOffset: number | null = null;
- if (!isCountQuery && page !== undefined) {
- const parsedPage = parseInt(String(page), 10);
- if (!isNaN(parsedPage) && parsedPage >= 1) {
- const pageOffset = (parsedPage - 1) * (validatedLimit || 100);
- validatedOffset = pageOffset;
- }
- }
- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);
if (parameter === "event_name") {
if (isCountQuery) {
@@ -418,45 +404,17 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole
`;
};
-export async function getMetric(req: FastifyRequest<GetMetricRequest>, res: FastifyReply) {
- const { parameter, page } = req.query;
- const site = req.params.siteId;
-
- const isPaginatedRequest = page !== undefined;
+export const getMetric = analyticsRoute<GetMetricRequest>(
+ req => req.query.parameter,
+ async (req: FastifyRequest<GetMetricRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const params = { siteId };
- const dataQuery = getQuery(req, false);
- const countQuery = getQuery(req, true);
-
- try {
- // Run both queries in parallel
- const [dataResult, countResult] = await Promise.all([
- clickhouse.query({
- query: dataQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- }),
- clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- }),
- ]);
-
- const items = await processResults<MetricItem>(dataResult);
- const countData = await processResults<{ totalCount: number }>(countResult);
- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;
+ const result = await runPaginatedQuery<MetricItem>(
+ { query: buildMetricQuery(req.query, siteId, false), params },
+ { query: buildMetricQuery(req.query, siteId, true), params }
+ );
- return res.send({ data: { data: items, totalCount } });
- } catch (error) {
- console.error(`Error fetching ${parameter}:`, error);
- console.error("Failed dataQuery:", dataQuery);
- if (isPaginatedRequest) {
- console.error("Failed countQuery:", countQuery);
- }
- return res.status(500).send({ error: `Failed to fetch ${parameter}` });
+ return res.send({ data: result });
}
-}
+);
diff --git a/server/src/api/analytics/getOrgEventCount.ts b/server/src/api/analytics/getOrgEventCount.ts
index 5723aedc6..757bd1027 100644
--- a/server/src/api/analytics/getOrgEventCount.ts
+++ b/server/src/api/analytics/getOrgEventCount.ts
@@ -1,8 +1,7 @@
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { getSitesUserHasAccessTo } from "../../lib/auth-utils.js";
-import { processResults } from "./utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
type OrgEventCountResponse = {
event_date: string;
@@ -18,42 +17,27 @@ type OrgEventCountResponse = {
event_count: number;
}[];
-export async function getOrgEventCount(
- req: FastifyRequest<{
- Params: {
- organizationId: string;
- };
- Querystring: {
- start_date?: string;
- end_date?: string;
- time_zone?: string;
- };
- }>,
- res: FastifyReply
-) {
- const { organizationId } = req.params;
- const { start_date, end_date, time_zone = "UTC" } = req.query;
-
- try {
- // Get all sites the user has access to
- const userSites = await getSitesUserHasAccessTo(req);
-
- // Filter to only sites in the requested organization
- const orgSites = userSites.filter(site => site.organizationId === organizationId);
-
- if (orgSites.length === 0) {
- return res.status(403).send({ error: "No access to organization or no sites found" });
- }
+interface GetOrgEventCountRequest {
+ Params: {
+ organizationId: string;
+ };
+ Querystring: {
+ start_date?: string;
+ end_date?: string;
+ time_zone?: string;
+ };
+}
- const siteIds = orgSites.map((site: any) => site.siteId);
+export const buildOrgEventCountQuery = (query: GetOrgEventCountRequest["Querystring"], siteIds: number[]) => {
+ const { start_date, end_date, time_zone = "UTC" } = query;
- // Build time filter for the query
- let timeFilter = "";
- let fillFromDate = "";
- let fillToDate = "";
+ // Build time filter for the query
+ let timeFilter = "";
+ let fillFromDate = "";
+ let fillToDate = "";
- if (start_date && end_date) {
- timeFilter = `AND event_hour >= toTimeZone(
+ if (start_date && end_date) {
+ timeFilter = `AND event_hour >= toTimeZone(
toStartOfDay(toDateTime(${SqlString.escape(start_date)}, ${SqlString.escape(time_zone)})),
'UTC'
)
@@ -66,13 +50,13 @@ export async function getOrgEventCount(
)
)`;
- // Set up WITH FILL parameters
- fillFromDate = `FROM toTimeZone(
+ // Set up WITH FILL parameters
+ fillFromDate = `FROM toTimeZone(
toStartOfDay(toDateTime(${SqlString.escape(start_date)}, ${SqlString.escape(time_zone)})),
'UTC'
)`;
- fillToDate = `TO if(
+ fillToDate = `TO if(
toDate(${SqlString.escape(end_date)}) = toDate(now(), ${SqlString.escape(time_zone)}),
toStartOfDay(now()) + INTERVAL 1 DAY,
toTimeZone(
@@ -80,14 +64,14 @@ export async function getOrgEventCount(
'UTC'
)
)`;
- } else {
- // No date range: return all data without WITH FILL
- timeFilter = "";
- fillFromDate = "";
- fillToDate = "";
- }
+ } else {
+ // No date range: return all data without WITH FILL
+ timeFilter = "";
+ fillFromDate = "";
+ fillToDate = "";
+ }
- const query = `
+ return `
SELECT
toStartOfDay(timestamp) as event_date,
countIf(type = 'pageview') as pageview_count,
@@ -108,16 +92,28 @@ export async function getOrgEventCount(
ORDER BY event_date
${fillFromDate ? `WITH FILL ${fillFromDate} ${fillToDate} STEP INTERVAL 1 DAY` : ""}
`;
+};
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- });
+export const getOrgEventCount = analyticsRoute<GetOrgEventCountRequest>(
+ "organization event count",
+ async (req: FastifyRequest<GetOrgEventCountRequest>, res: FastifyReply) => {
+ const { organizationId } = req.params;
- const data = await processResults<OrgEventCountResponse[number]>(result);
+ // Get all sites the user has access to
+ const userSites = await getSitesUserHasAccessTo(req);
+
+ // Filter to only sites in the requested organization
+ const orgSites = userSites.filter(site => site.organizationId === organizationId);
+
+ if (orgSites.length === 0) {
+ return res.status(403).send({ error: "No access to organization or no sites found" });
+ }
+
+ const siteIds = orgSites.map((site: any) => site.siteId);
+
+ const data = await runAnalyticsQuery<OrgEventCountResponse[number]>({
+ query: buildOrgEventCountQuery(req.query, siteIds),
+ });
return res.send({ data });
- } catch (error) {
- console.error("Error fetching organization event count:", error);
- return res.status(500).send({ error: "Failed to fetch organization event count" });
}
-}
+);
diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.ts
index fbd47a855..fa83b3cfc 100644
--- a/server/src/api/analytics/getOverview.ts
+++ b/server/src/api/analytics/getOverview.ts
@@ -1,8 +1,8 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { getFilterStatement } from "./utils/getFilterStatement.js";
-import { getTimeStatement, processResults } from "./utils/utils.js";
+import { getTimeStatement } from "./utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
type GetOverviewResponse = {
sessions: number;
@@ -13,7 +13,7 @@ type GetOverviewResponse = {
session_duration: number;
};
-const getQuery = (params: FilterParams, siteId: number) => {
+export const buildOverviewQuery = (params: FilterParams, siteId: number) => {
const timeStatement = getTimeStatement(params);
const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);
@@ -61,46 +61,16 @@ export interface OverviewRequest {
Querystring: FilterParams;
}
-export async function getOverview(req: FastifyRequest<OverviewRequest>, res: FastifyReply) {
- const {
- start_date,
- end_date,
- time_zone,
- filters,
- start_datetime,
- end_datetime,
- past_minutes_start,
- past_minutes_end,
- } = req.query;
- const site = req.params.siteId;
+export const getOverview = analyticsRoute<OverviewRequest>(
+ "overview",
+ async (req: FastifyRequest<OverviewRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
- const query = getQuery(
- {
- start_date,
- end_date,
- time_zone,
- filters,
- start_datetime,
- end_datetime,
- past_minutes_start,
- past_minutes_end,
- },
- Number(site)
- );
-
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ const data = await runAnalyticsQuery<GetOverviewResponse>({
+ query: buildOverviewQuery(req.query, siteId),
+ params: { siteId },
});
- const data = await processResults<GetOverviewResponse>(result);
return res.send({ data: data[0] });
- } catch (error) {
- console.error("Error fetching overview:", error);
- return res.status(500).send({ error: "Failed to fetch overview" });
}
-}
+);
diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.ts
index 3fb8af8ac..730fb1a05 100644
--- a/server/src/api/analytics/getOverviewBucketed.ts
+++ b/server/src/api/analytics/getOverviewBucketed.ts
@@ -1,17 +1,16 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { validateTimeStatementFillParams } from "./utils/query-validation.js";
import {
getTimeStatement,
normalizeDatetimeForClickhouse,
- processResults,
TimeBucketToFn,
bucketIntervalMap,
} from "./utils/utils.js";
import { getFilterStatement } from "./utils/getFilterStatement.js";
import { TimeBucket } from "./types.js";
+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);
@@ -89,7 +88,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
return "";
}
-const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {
+export const buildOverviewBucketedQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {
const {
start_date,
end_date,
@@ -183,32 +182,19 @@ ORDER BY time`;
type getOverviewBucketed = { time: string; pageviews: number }[];
-export async function getOverviewBucketed(
- req: FastifyRequest<{
- Params: {
- siteId: string;
- };
- Querystring: FilterParams<{
- bucket: TimeBucket;
- }>;
- }>,
- res: FastifyReply
-) {
- const {
- start_date,
- end_date,
- time_zone,
- bucket,
- filters,
- start_datetime,
- end_datetime,
- past_minutes_start,
- past_minutes_end,
- } = req.query;
- const site = req.params.siteId;
+interface GetOverviewBucketedRequest {
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{
+ bucket: TimeBucket;
+ }>;
+}
- const query = getQuery(
- {
+export const getOverviewBucketed = analyticsRoute<GetOverviewBucketedRequest>(
+ "pageviews",
+ async (req: FastifyRequest<GetOverviewBucketedRequest>, res: FastifyReply) => {
+ const {
start_date,
end_date,
time_zone,
@@ -218,23 +204,27 @@ export async function getOverviewBucketed(
end_datetime,
past_minutes_start,
past_minutes_end,
- },
- Number(site)
- );
+ } = req.query;
+ const site = req.params.siteId;
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ const data = await runAnalyticsQuery<getOverviewBucketed[number]>({
+ query: buildOverviewBucketedQuery(
+ {
+ start_date,
+ end_date,
+ time_zone,
+ bucket,
+ filters,
+ start_datetime,
+ end_datetime,
+ past_minutes_start,
+ past_minutes_end,
+ },
+ Number(site)
+ ),
+ params: { siteId: Number(site) },
});
- const data = await processResults<getOverviewBucketed[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching pageviews:", error);
- return res.status(500).send({ error: "Failed to fetch pageviews" });
}
-}
+);
diff --git a/server/src/api/analytics/getPageTitles.ts b/server/src/api/analytics/getPageTitles.ts
index 88ddacd00..5d820c1cc 100644
--- a/server/src/api/analytics/getPageTitles.ts
+++ b/server/src/api/analytics/getPageTitles.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "./utils/utils.js";
+import { getTimeStatement } from "./utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "./utils/getFilterStatement.js";
+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";
interface GetPageTitlesRequest {
Params: {
@@ -31,31 +31,18 @@ type PageTitlesPaginatedResponse = {
totalCount: number;
};
-const getPageTitlesQuery = (request: FastifyRequest<GetPageTitlesRequest>, isCountQuery: boolean = false) => {
- const { filters, limit, page } = request.query;
+export const buildPageTitlesQuery = (
+ query: GetPageTitlesRequest["Querystring"],
+ siteId: number,
+ isCountQuery: boolean = false
+) => {
+ const { filters } = query;
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
- let validatedLimit: number | null = null;
- if (!isCountQuery && limit !== undefined) {
- const parsedLimit = parseInt(String(limit), 10);
- if (!isNaN(parsedLimit) && parsedLimit > 0) {
- validatedLimit = parsedLimit;
- }
- }
- // StandardSection usually shows a small number, e.g., 7 or 10. Let's default to 10 for non-paginated use.
- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 10";
-
- let validatedOffset: number | null = null;
- if (!isCountQuery && page !== undefined) {
- const parsedPage = parseInt(String(page), 10);
- if (!isNaN(parsedPage) && parsedPage >= 1) {
- const pageOffset = (parsedPage - 1) * (validatedLimit || 10);
- validatedOffset = pageOffset;
- }
- }
- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";
+ // StandardSection usually shows a small number, e.g., 7 or 10. Default to 10 for non-paginated use.
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 10, isCountQuery);
// For page_title, we want to count distinct sessions that viewed this title.
// We also need a representative pathname and calculate average time on page.
@@ -143,47 +130,23 @@ const getPageTitlesQuery = (request: FastifyRequest<GetPageTitlesRequest>, isCou
`;
};
-export async function getPageTitles(req: FastifyRequest<GetPageTitlesRequest>, res: FastifyReply) {
- const site = req.params.siteId;
- const { page } = req.query;
-
- const isPaginatedRequest = page !== undefined; // True if page is present
+export const getPageTitles = analyticsRoute<GetPageTitlesRequest>(
+ "page titles",
+ async (req: FastifyRequest<GetPageTitlesRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const params = { siteId };
+ const dataSpec = { query: buildPageTitlesQuery(req.query, siteId, false), params };
- const dataQuery = getPageTitlesQuery(req, false);
-
- try {
- const dataResult = await clickhouse.query({
- query: dataQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- });
- const items = await processResults<PageTitleItem>(dataResult);
-
- if (isPaginatedRequest) {
- const countQuery = getPageTitlesQuery(req, true);
- const countResult = await clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ if (req.query.page !== undefined) {
+ const result = await runPaginatedQuery<PageTitleItem>(dataSpec, {
+ query: buildPageTitlesQuery(req.query, siteId, true),
+ params,
});
- const countData = await processResults<{ totalCount: number }>(countResult);
- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;
- return res.send({ data: { data: items, totalCount } });
- } else {
- // For non-paginated (StandardSection default) use, return the simpler structure
- return res.send({ data: items });
- }
- } catch (error) {
- console.error(`Error fetching page titles:`, error);
- console.error("Failed dataQuery:", dataQuery);
- if (isPaginatedRequest) {
- const countQuery = getPageTitlesQuery(req, true);
- console.error("Failed countQuery:", countQuery);
+ return res.send({ data: result });
}
- return res.status(500).send({ error: `Failed to fetch page titles` });
+
+ // For non-paginated (StandardSection default) use, return the simpler structure
+ const items = await runAnalyticsQuery<PageTitleItem>(dataSpec);
+ return res.send({ data: items });
}
-}
+);
diff --git a/server/src/api/analytics/getRetention.ts b/server/src/api/analytics/getRetention.ts
index 07d1a374f..9b56d39aa 100644
--- a/server/src/api/analytics/getRetention.ts
+++ b/server/src/api/analytics/getRetention.ts
@@ -1,6 +1,5 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../db/clickhouse/clickhouse.js";
-import { processResults } from "./utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";
// Define the expected shape of a single data row from the query
interface RetentionDataRow {
@@ -19,28 +18,17 @@ interface ProcessedRetentionData {
range: number;
}
-export const getRetention = async (
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: { mode?: string; range?: string };
- }>,
- res: FastifyReply
-) => {
- const { siteId } = req.params;
- const { mode = "week", range = "90" } = req.query; // Default to weekly mode and 90 days range
-
- // Validate mode parameter
- const retentionMode = mode === "day" ? "day" : "week";
-
- // Validate range parameter (between 7-365 days)
- const timeRange = Math.min(365, Math.max(7, parseInt(range) || 90));
+interface GetRetentionRequest {
+ Params: { siteId: string };
+ Querystring: { mode?: string; range?: string };
+}
+export const buildRetentionQuery = (retentionMode: "day" | "week") => {
// Build the appropriate SQL based on the retention mode
const periodFunction = retentionMode === "day" ? "toDate" : "toStartOfWeek";
const periodDiffFunc = retentionMode === "day" ? "day" : "week";
- const query = await clickhouse.query({
- query: `
+ return `
WITH UserFirstPeriod AS (
SELECT
-- Use effective user ID: identified_user_id for identified users, user_id for anonymous
@@ -91,26 +79,40 @@ JOIN CohortSize cs ON cr.cohort_period = cs.cohort_period
ORDER BY
cr.cohort_period DESC,
cr.period_difference ASC;
- `,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(siteId),
- timeRange: timeRange,
- },
- });
-
- const results = await processResults<RetentionDataRow>(query);
-
- // Process data into a grid-friendly format
- const processedData: ProcessedRetentionData = {
- ...processRetentionData(results),
- mode: retentionMode as "day" | "week",
- range: timeRange,
- };
-
- return res.send({ data: processedData });
+ `;
};
+export const getRetention = analyticsRoute<GetRetentionRequest>(
+ "retention",
+ async (req: FastifyRequest<GetRetentionRequest>, res: FastifyReply) => {
+ const { siteId } = req.params;
+ const { mode = "week", range = "90" } = req.query; // Default to weekly mode and 90 days range
+
+ // Validate mode parameter
+ const retentionMode = mode === "day" ? "day" : "week";
+
+ // Validate range parameter (between 7-365 days)
+ const timeRange = Math.min(365, Math.max(7, parseInt(range) || 90));
+
+ const results = await runAnalyticsQuery<RetentionDataRow>({
+ query: buildRetentionQuery(retentionMode),
+ params: {
+ siteId: Number(siteId),
+ timeRange: timeRange,
+ },
+ });
+
+ // Process data into a grid-friendly format
+ const processedData: ProcessedRetentionData = {
+ ...processRetentionData(results),
+ mode: retentionMode as "day" | "week",
+ range: timeRange,
+ };
+
+ return res.send({ data: processedData });
+ }
+);
+
// Process raw retention data into a grid-friendly format
function processRetentionData(rawData: RetentionDataRow[]): Omit<ProcessedRetentionData, "mode" | "range"> {
if (!rawData || rawData.length === 0) {
diff --git a/server/src/api/analytics/goals/getGoalSessions.ts b/server/src/api/analytics/goals/getGoalSessions.ts
index b7e62312c..275309643 100644
--- a/server/src/api/analytics/goals/getGoalSessions.ts
+++ b/server/src/api/analytics/goals/getGoalSessions.ts
@@ -1,11 +1,11 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { db } from "../../../db/postgres/postgres.js";
import { goals } from "../../../db/postgres/schema.js";
import { eq } from "drizzle-orm";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { GetSessionsResponse } from "../sessions/getSessions.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { buildGoalCondition } from "./goalConditions.js";
export interface GetGoalSessionsRequest {
@@ -19,40 +19,16 @@ export interface GetGoalSessionsRequest {
}>;
}
-export async function getGoalSessions(req: FastifyRequest<GetGoalSessionsRequest>, res: FastifyReply) {
- const { goalId, siteId } = req.params;
- const { page, limit } = req.query;
+export const buildGoalSessionsQuery = (
+ query: GetGoalSessionsRequest["Querystring"],
+ siteId: number,
+ goalCondition: string
+) => {
+ const timeStatement = getTimeStatement(query);
- try {
- // Fetch the goal from PostgreSQL
- const goal = await db
- .select()
- .from(goals)
- .where(eq(goals.goalId, Number(goalId)))
- .limit(1);
-
- if (!goal || goal.length === 0) {
- return res.status(404).send({ error: "Goal not found" });
- }
-
- const goalData = goal[0];
-
- // Verify the goal belongs to the site
- if (goalData.siteId !== Number(siteId)) {
- return res.status(403).send({ error: "Goal does not belong to this site" });
- }
-
- const timeStatement = getTimeStatement(req.query);
-
- // Build the goal matching condition
- const goalCondition = buildGoalCondition(goalData);
- if (!goalCondition) {
- return res.status(400).send({ error: "Invalid goal configuration" });
- }
-
- // Build query to find sessions that match the goal
- // First, find all session_ids that have at least one event matching the goal
- const query = `
+ // Build query to find sessions that match the goal
+ // First, find all session_ids that have at least one event matching the goal
+ return `
WITH GoalSessions AS (
SELECT DISTINCT session_id
FROM events
@@ -113,21 +89,47 @@ export async function getGoalSessions(req: FastifyRequest<GetGoalSessionsRequest
FROM AggregatedSessions
LIMIT {limit:Int32} OFFSET {offset:Int32}
`;
+};
+
+export const getGoalSessions = analyticsRoute<GetGoalSessionsRequest>(
+ "goal sessions",
+ async (req: FastifyRequest<GetGoalSessionsRequest>, res: FastifyReply) => {
+ const { goalId, siteId } = req.params;
+ const { page, limit } = req.query;
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ // Fetch the goal from PostgreSQL
+ const goal = await db
+ .select()
+ .from(goals)
+ .where(eq(goals.goalId, Number(goalId)))
+ .limit(1);
+
+ if (!goal || goal.length === 0) {
+ return res.status(404).send({ error: "Goal not found" });
+ }
+
+ const goalData = goal[0];
+
+ // Verify the goal belongs to the site
+ if (goalData.siteId !== Number(siteId)) {
+ return res.status(403).send({ error: "Goal does not belong to this site" });
+ }
+
+ // Build the goal matching condition
+ const goalCondition = buildGoalCondition(goalData);
+ if (!goalCondition) {
+ return res.status(400).send({ error: "Invalid goal configuration" });
+ }
+
+ const data = await runAnalyticsQuery<GetSessionsResponse[number]>({
+ query: buildGoalSessionsQuery(req.query, Number(siteId), goalCondition),
+ params: {
siteId: Number(siteId),
limit: limit || 25,
offset: ((page || 1) - 1) * (limit || 25),
},
});
- const data = await processResults<GetSessionsResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching goal sessions:", error);
- return res.status(500).send({ error: "Failed to fetch goal sessions" });
}
-}
+);
diff --git a/server/src/api/analytics/goals/getGoalTimeSeries.ts b/server/src/api/analytics/goals/getGoalTimeSeries.ts
index b806c37fa..d2764b1d3 100644
--- a/server/src/api/analytics/goals/getGoalTimeSeries.ts
+++ b/server/src/api/analytics/goals/getGoalTimeSeries.ts
@@ -2,12 +2,12 @@ import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import { and, inArray, eq } from "drizzle-orm";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { db } from "../../../db/postgres/postgres.js";
import { goals } from "../../../db/postgres/schema.js";
import { TimeBucket } from "../types.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";
+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { buildGoalCondition } from "./goalConditions.js";
type GoalTimeSeriesPoint = {
@@ -47,40 +47,26 @@ const parseGoalIds = (value: string | number[] | undefined) => {
return value.split(",").map(Number).filter(Number.isFinite);
};
-export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesRequest>, res: FastifyReply) {
- const siteId = Number(req.params.siteId);
- const { bucket = "hour" } = req.query;
- const timeZone = req.query.time_zone || "UTC";
-
- if (!TimeBucketToFn[bucket]) {
- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
- }
-
- const goalIds = [...new Set(parseGoalIds(req.query.goal_ids))];
- if (goalIds.length === 0) {
- return res.send({ data: [] });
- }
-
- try {
- const siteGoals = await db
- .select()
- .from(goals)
- .where(and(eq(goals.siteId, siteId), inArray(goals.goalId, goalIds)));
-
- if (siteGoals.length === 0) {
- return res.send({ data: [] });
- }
-
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(req.query.filters, siteId, timeStatement);
- const bucketFn = TimeBucketToFn[bucket];
-
- const conversionQueries = siteGoals
- .map(goal => {
- const goalCondition = buildGoalCondition(goal);
- if (!goalCondition) return null;
-
- return `
+/**
+ * Returns null when none of the site goals produces a valid conversion
+ * condition — the handler responds with empty data in that case.
+ */
+export const buildGoalTimeSeriesQuery = (
+ query: GetGoalTimeSeriesRequest["Querystring"],
+ siteId: number,
+ siteGoals: (typeof goals.$inferSelect)[]
+): string | null => {
+ const { bucket = "hour" } = query;
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
+ const bucketFn = TimeBucketToFn[bucket];
+
+ const conversionQueries = siteGoals
+ .map(goal => {
+ const goalCondition = buildGoalCondition(goal);
+ if (!goalCondition) return null;
+
+ return `
SELECT
toDateTime(${bucketFn}(toTimeZone(timestamp, {timeZone:String}))) AS time,
${SqlString.escape(goal.goalId)} AS goal_id,
@@ -93,15 +79,15 @@ export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesReq
${filterStatement}
GROUP BY time, goal_id
`;
- })
- .filter(Boolean);
+ })
+ .filter(Boolean);
- if (conversionQueries.length === 0) {
- return res.send({ data: [] });
- }
+ if (conversionQueries.length === 0) {
+ return null;
+ }
- const validGoalIds = siteGoals.map(goal => goal.goalId).join(", ");
- const query = `
+ const validGoalIds = siteGoals.map(goal => goal.goalId).join(", ");
+ return `
WITH
sessions_by_bucket AS (
SELECT
@@ -131,20 +117,46 @@ export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesReq
LEFT JOIN conversions_by_goal c ON c.time = s.time AND c.goal_id = g.goal_id
ORDER BY s.time ASC, g.goal_id ASC
`;
+};
+
+export const getGoalTimeSeries = analyticsRoute<GetGoalTimeSeriesRequest>(
+ "goal time series",
+ async (req: FastifyRequest<GetGoalTimeSeriesRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const { bucket = "hour" } = req.query;
+ const timeZone = req.query.time_zone || "UTC";
+
+ if (!TimeBucketToFn[bucket]) {
+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });
+ }
+
+ const goalIds = [...new Set(parseGoalIds(req.query.goal_ids))];
+ if (goalIds.length === 0) {
+ return res.send({ data: [] });
+ }
+
+ const siteGoals = await db
+ .select()
+ .from(goals)
+ .where(and(eq(goals.siteId, siteId), inArray(goals.goalId, goalIds)));
+
+ if (siteGoals.length === 0) {
+ return res.send({ data: [] });
+ }
+
+ const query = buildGoalTimeSeriesQuery(req.query, siteId, siteGoals);
+ if (query === null) {
+ return res.send({ data: [] });
+ }
- const result = await clickhouse.query({
+ const data = await runAnalyticsQuery<GoalTimeSeriesPoint>({
query,
- format: "JSONEachRow",
- query_params: {
+ params: {
siteId,
timeZone,
},
});
- const data = await processResults<GoalTimeSeriesPoint>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching goal time series:", error);
- return res.status(500).send({ error: "Failed to fetch goal time series" });
}
-}
+);
diff --git a/server/src/api/analytics/goals/getGoals.ts b/server/src/api/analytics/goals/getGoals.ts
index fc6624b49..a0410af9b 100644
--- a/server/src/api/analytics/goals/getGoals.ts
+++ b/server/src/api/analytics/goals/getGoals.ts
@@ -1,12 +1,12 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { db } from "../../../db/postgres/postgres.js";
import { goals } from "../../../db/postgres/schema.js";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { eq, desc, asc, sql } from "drizzle-orm";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import SqlString from "sqlstring";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { buildGoalCondition } from "./goalConditions.js";
// Types for the response
@@ -31,36 +31,92 @@ interface GetGoalsResponse {
};
}
-export async function getGoals(
- request: FastifyRequest<{
- Params: {
- siteId: string;
- };
- Querystring: FilterParams<{
- page?: string;
- page_size?: string;
- sort?: string;
- order?: "asc" | "desc";
- }>;
- }>,
- reply: FastifyReply
-) {
- const { siteId } = request.params;
- const { filters, page = "1", page_size: pageSize = "10", sort = "createdAt", order = "desc" } = request.query;
-
- const pageNumber = parseInt(page, 10);
- const pageSizeNumber = parseInt(pageSize, 10);
-
- // Validate page and pageSize
- if (isNaN(pageNumber) || pageNumber < 1) {
- return reply.status(400).send({ error: "Invalid page number" });
+interface GetGoalsRequest {
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{
+ page?: string;
+ page_size?: string;
+ sort?: string;
+ order?: "asc" | "desc";
+ }>;
+}
+
+export const buildGoalsTotalSessionsQuery = (query: GetGoalsRequest["Querystring"], siteId: number) => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = query.filters ? getFilterStatement(query.filters, siteId, timeStatement) : "";
+
+ return `
+ SELECT COUNT(DISTINCT session_id) AS total_sessions
+ FROM events
+ WHERE site_id = ${SqlString.escape(siteId)}
+ ${timeStatement}
+ ${filterStatement}
+ `;
+};
+
+/**
+ * Returns null when none of the goals produces a valid conversion condition —
+ * the handler responds with zero-conversion goals in that case.
+ */
+export const buildGoalsConversionsQuery = (
+ query: GetGoalsRequest["Querystring"],
+ siteId: number,
+ siteGoals: (typeof goals.$inferSelect)[]
+): string | null => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = query.filters ? getFilterStatement(query.filters, siteId, timeStatement) : "";
+
+ // Build a single query that calculates all goal conversions at once using conditional aggregation
+ // This is more efficient than separate queries for each goal
+ let conditionalClauses: string[] = [];
+
+ for (const goal of siteGoals) {
+ const goalCondition = buildGoalCondition(goal);
+ if (!goalCondition) continue;
+
+ conditionalClauses.push(`
+ COUNT(DISTINCT IF(
+ ${goalCondition},
+ session_id,
+ NULL
+ )) AS goal_${goal.goalId}_conversions
+ `);
}
- if (isNaN(pageSizeNumber) || pageSizeNumber < 1 || pageSizeNumber > 100) {
- return reply.status(400).send({ error: "Invalid page size, must be between 1 and 100" });
+ if (conditionalClauses.length === 0) {
+ return null;
}
- try {
+ return `
+ SELECT
+ ${conditionalClauses.join(", ")}
+ FROM events
+ WHERE site_id = ${SqlString.escape(siteId)}
+ ${timeStatement}
+ ${filterStatement}
+ `;
+};
+
+export const getGoals = analyticsRoute<GetGoalsRequest>(
+ "goals data",
+ async (request: FastifyRequest<GetGoalsRequest>, reply: FastifyReply) => {
+ const { siteId } = request.params;
+ const { page = "1", page_size: pageSize = "10", sort = "createdAt", order = "desc" } = request.query;
+
+ const pageNumber = parseInt(page, 10);
+ const pageSizeNumber = parseInt(pageSize, 10);
+
+ // Validate page and pageSize
+ if (isNaN(pageNumber) || pageNumber < 1) {
+ return reply.status(400).send({ error: "Invalid page number" });
+ }
+
+ if (isNaN(pageSizeNumber) || pageSizeNumber < 1 || pageSizeNumber > 100) {
+ return reply.status(400).send({ error: "Invalid page size, must be between 1 and 100" });
+ }
+
// Count total goals for pagination metadata
const totalGoalsResult = await db
.select({ count: sql<number>`count(*)` })
@@ -123,45 +179,15 @@ export async function getGoals(
});
}
- // Build filter and time clauses for ClickHouse queries
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = filters ? getFilterStatement(filters, Number(siteId), timeStatement) : "";
-
// First, get the total number of unique sessions (denominator for conversion rate)
- const totalSessionsQuery = `
- SELECT COUNT(DISTINCT session_id) AS total_sessions
- FROM events
- WHERE site_id = ${SqlString.escape(Number(siteId))}
- ${timeStatement}
- ${filterStatement}
- `;
-
- const totalSessionsResult = await clickhouse.query({
- query: totalSessionsQuery,
- format: "JSONEachRow",
+ const totalSessionsData = await runAnalyticsQuery<{ total_sessions: number }>({
+ query: buildGoalsTotalSessionsQuery(request.query, Number(siteId)),
});
-
- const totalSessionsData = await processResults<{ total_sessions: number }>(totalSessionsResult);
const totalSessions = totalSessionsData[0]?.total_sessions || 0;
- // Build a single query that calculates all goal conversions at once using conditional aggregation
- // This is more efficient than separate queries for each goal
- let conditionalClauses: string[] = [];
-
- for (const goal of siteGoals) {
- const goalCondition = buildGoalCondition(goal);
- if (!goalCondition) continue;
+ const conversionQuery = buildGoalsConversionsQuery(request.query, Number(siteId), siteGoals);
- conditionalClauses.push(`
- COUNT(DISTINCT IF(
- ${goalCondition},
- session_id,
- NULL
- )) AS goal_${goal.goalId}_conversions
- `);
- }
-
- if (conditionalClauses.length === 0) {
+ if (conversionQuery === null) {
// If no valid goals to calculate, return the goals without conversion data
const goalsWithZeroConversions = siteGoals.map(goal => ({
...goal,
@@ -182,21 +208,7 @@ export async function getGoals(
}
// Execute the comprehensive query
- const conversionQuery = `
- SELECT
- ${conditionalClauses.join(", ")}
- FROM events
- WHERE site_id = ${SqlString.escape(Number(siteId))}
- ${timeStatement}
- ${filterStatement}
- `;
-
- const conversionResult = await clickhouse.query({
- query: conversionQuery,
- format: "JSONEachRow",
- });
-
- const conversionData = await processResults<Record<string, number>>(conversionResult);
+ const conversionData = await runAnalyticsQuery<Record<string, number>>({ query: conversionQuery });
// If we didn't get any results, use zeros
const conversions = conversionData[0] || {};
@@ -223,8 +235,5 @@ export async function getGoals(
totalPages,
},
});
- } catch (error) {
- console.error("Error fetching goals:", error);
- return reply.status(500).send({ error: "Failed to fetch goals data" });
}
-}
+);
diff --git a/server/src/api/analytics/lite/getMetricLite.ts b/server/src/api/analytics/lite/getMetricLite.ts
index 24018753a..923565ff3 100644
--- a/server/src/api/analytics/lite/getMetricLite.ts
+++ b/server/src/api/analytics/lite/getMetricLite.ts
@@ -1,8 +1,7 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getMetric } from "../getMetric.js";
-import { processResults } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters } from "./utils.js";
// Lite metric supports only dimensions backed by MVs:
@@ -24,43 +23,29 @@ type LiteMetricItem = {
total_count?: number;
};
-export async function getMetricLite(
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: FilterParams<{ parameter: LiteMetricParameter; limit?: number; page?: number }>;
- }>,
- res: FastifyReply
-) {
- const site = Number(req.params.siteId);
- const { parameter } = req.query;
- const limit = Math.min(Number(req.query.limit) || 250, 500);
- const page = Math.max(Number(req.query.page) || 1, 1);
- const offsetStatement = page > 1 ? `OFFSET ${(page - 1) * limit}` : "";
- const filtersPresent = hasLiteFilters(req.query.filters);
-
- // Pull totalCount off the window column and drop it from the returned rows so
- // items match the standard /metric shape.
- const sendMetric = (data: LiteMetricItem[]) => {
- const totalCount = data.length > 0 ? (data[0].total_count ?? data.length) : 0;
- const items = data.map(({ total_count, ...rest }) => rest);
- return res.send({ data: { data: items, totalCount } });
- };
+interface GetMetricLiteRequest {
+ Params: { siteId: string };
+ Querystring: FilterParams<{ parameter: LiteMetricParameter; limit?: number; page?: number }>;
+}
- // Filtered country/device_type lists can be served from sessions_mv_target
- // (it carries both columns per session), as long as every active filter
- // targets a session column. pathname isn't on the session rollup — and no
- // dimensioned MV pairs pathname with another dimension — so any filtered
- // pathname list falls back to the raw-events query.
- if (filtersPresent) {
- const filter = getLiteSessionFilter(req.query.filters);
- const sessionFastPath = filter.supported && (parameter === "country" || parameter === "device_type");
- if (!sessionFastPath) {
- return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);
- }
+// Lite pagination clamps rather than validates (default 250, max 500), so it
+// keeps its own logic instead of getPaginationStatements.
+const getLitePagination = (query: GetMetricLiteRequest["Querystring"]) => {
+ const limit = Math.min(Number(query.limit) || 250, 500);
+ const page = Math.max(Number(query.page) || 1, 1);
+ const offsetStatement = page > 1 ? `OFFSET ${(page - 1) * limit}` : "";
+ return { limit, offsetStatement };
+};
- const sessionsTime = getLiteTimeStatement(req.query, "start_time");
- const nonEmpty = parameter === "device_type" ? `AND ${parameter} <> ''` : "";
- const query = `
+// Filtered country/device_type lists can be served from sessions_mv_target
+// (it carries both columns per session), as long as every active filter
+// targets a session column.
+export const buildMetricLiteSessionQuery = (query: GetMetricLiteRequest["Querystring"], filterSql: string) => {
+ const { parameter } = query;
+ const { limit, offsetStatement } = getLitePagination(query);
+ const sessionsTime = getLiteTimeStatement(query, "start_time");
+ const nonEmpty = parameter === "device_type" ? `AND ${parameter} <> ''` : "";
+ return `
SELECT
value,
pageviews,
@@ -82,7 +67,7 @@ export async function getMetricLite(
WHERE site_id = {siteId:Int32}
${sessionsTime}
${nonEmpty}
- ${filter.sql}
+ ${filterSql}
GROUP BY session_id
)
GROUP BY ${parameter}
@@ -91,29 +76,20 @@ export async function getMetricLite(
LIMIT ${limit}
${offsetStatement}
`;
+};
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: { siteId: site },
- });
- const data = await processResults<LiteMetricItem>(result);
- return sendMetric(data);
- } catch (error) {
- console.error("Error fetching lite metric:", error);
- return res.status(500).send({ error: "Failed to fetch metric" });
- }
- }
-
- const timeStatement = getLiteTimeStatement(req.query, "event_hour");
+// Unfiltered lists read the dimensioned hourly MVs directly. Returns null for
+// parameters lite mode doesn't support — the handler responds 400.
+export const buildMetricLiteQuery = (query: GetMetricLiteRequest["Querystring"]): string | null => {
+ const { parameter } = query;
+ const { limit, offsetStatement } = getLitePagination(query);
+ const timeStatement = getLiteTimeStatement(query, "event_hour");
// Percentages are computed in an outer pass so the window function
// operates on already-grouped rows. `sum(sum(...)) OVER ()` is illegal
// in ClickHouse — aggregates can't nest.
- let query: string;
if (parameter === "pathname") {
- query = `
+ return `
SELECT
value,
hostname,
@@ -137,8 +113,9 @@ export async function getMetricLite(
LIMIT ${limit}
${offsetStatement}
`;
- } else if (parameter === "country") {
- query = `
+ }
+ if (parameter === "country") {
+ return `
SELECT
value,
pageviews,
@@ -160,8 +137,9 @@ export async function getMetricLite(
LIMIT ${limit}
${offsetStatement}
`;
- } else if (parameter === "device_type") {
- query = `
+ }
+ if (parameter === "device_type") {
+ return `
SELECT
value,
pageviews,
@@ -184,20 +162,53 @@ export async function getMetricLite(
LIMIT ${limit}
${offsetStatement}
`;
- } else {
- return res.status(400).send({ error: "Lite mode does not support this parameter" });
}
+ return null;
+};
+
+export const getMetricLite = analyticsRoute<GetMetricLiteRequest>(
+ "metric",
+ async (req: FastifyRequest<GetMetricLiteRequest>, res: FastifyReply) => {
+ const site = Number(req.params.siteId);
+ const filtersPresent = hasLiteFilters(req.query.filters);
+
+ // Pull totalCount off the window column and drop it from the returned rows so
+ // items match the standard /metric shape.
+ const sendMetric = (data: LiteMetricItem[]) => {
+ const totalCount = data.length > 0 ? (data[0].total_count ?? data.length) : 0;
+ const items = data.map(({ total_count, ...rest }) => rest);
+ return res.send({ data: { data: items, totalCount } });
+ };
+
+ // Filtered country/device_type lists can be served from sessions_mv_target,
+ // as long as every active filter targets a session column. pathname isn't
+ // on the session rollup — and no dimensioned MV pairs pathname with another
+ // dimension — so any filtered pathname list falls back to the raw-events
+ // query.
+ if (filtersPresent) {
+ const filter = getLiteSessionFilter(req.query.filters);
+ const sessionFastPath =
+ filter.supported && (req.query.parameter === "country" || req.query.parameter === "device_type");
+ if (!sessionFastPath) {
+ return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);
+ }
+
+ const data = await runAnalyticsQuery<LiteMetricItem>({
+ query: buildMetricLiteSessionQuery(req.query, filter.sql),
+ params: { siteId: site },
+ });
+ return sendMetric(data);
+ }
- try {
- const result = await clickhouse.query({
+ const query = buildMetricLiteQuery(req.query);
+ if (query === null) {
+ return res.status(400).send({ error: "Lite mode does not support this parameter" });
+ }
+
+ const data = await runAnalyticsQuery<LiteMetricItem>({
query,
- format: "JSONEachRow",
- query_params: { siteId: site },
+ params: { siteId: site },
});
- const data = await processResults<LiteMetricItem>(result);
return sendMetric(data);
- } catch (error) {
- console.error("Error fetching lite metric:", error);
- return res.status(500).send({ error: "Failed to fetch metric" });
}
-}
+);
diff --git a/server/src/api/analytics/lite/getOverviewBucketedLite.ts b/server/src/api/analytics/lite/getOverviewBucketedLite.ts
index 30bd8713f..48b0cee73 100644
--- a/server/src/api/analytics/lite/getOverviewBucketedLite.ts
+++ b/server/src/api/analytics/lite/getOverviewBucketedLite.ts
@@ -1,10 +1,10 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import SqlString from "sqlstring";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getOverviewBucketed } from "../getOverviewBucketed.js";
import { TimeBucket } from "../types.js";
-import { TimeBucketToFn, processResults } from "../utils/utils.js";
+import { TimeBucketToFn } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { getLiteFillClause, getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters, liteBucket } from "./utils.js";
type GetOverviewBucketedLiteResponse = {
@@ -165,71 +165,66 @@ function buildSessionMvFilteredQuery(args: {
`;
}
-export async function getOverviewBucketedLite(
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: FilterParams<{ bucket: TimeBucket }>;
- }>,
- res: FastifyReply
-) {
- const site = Number(req.params.siteId);
- const bucket = liteBucket(req.query.bucket);
- const fn = TimeBucketToFn[bucket];
- const tz = SqlString.escape(req.query.time_zone || "UTC");
+interface GetOverviewBucketedLiteRequest {
+ Params: { siteId: string };
+ Querystring: FilterParams<{ bucket: TimeBucket }>;
+}
- const isAllTime =
- !req.query.start_date &&
- !req.query.end_date &&
- req.query.past_minutes_start === undefined &&
- req.query.past_minutes_end === undefined;
- const fill = isAllTime ? "" : getLiteFillClause(req.query, bucket);
+export const getOverviewBucketedLite = analyticsRoute<GetOverviewBucketedLiteRequest>(
+ "overview",
+ async (req: FastifyRequest<GetOverviewBucketedLiteRequest>, res: FastifyReply) => {
+ const site = Number(req.params.siteId);
+ const bucket = liteBucket(req.query.bucket);
+ const fn = TimeBucketToFn[bucket];
+ const tz = SqlString.escape(req.query.time_zone || "UTC");
- const filtersPresent = hasLiteFilters(req.query.filters);
+ const isAllTime =
+ !req.query.start_date &&
+ !req.query.end_date &&
+ req.query.past_minutes_start === undefined &&
+ req.query.past_minutes_end === undefined;
+ const fill = isAllTime ? "" : getLiteFillClause(req.query, bucket);
- let query: string;
- if (filtersPresent) {
- // Session-column filters stay on sessions_mv_target; anything else falls
- // back to the raw-events query.
- const filter = getLiteSessionFilter(req.query.filters);
- if (!filter.supported) {
- return getOverviewBucketed(req, res);
- }
- query = buildSessionMvFilteredQuery({
- fn,
- tz,
- sessionsTime: getLiteTimeStatement(req.query, "start_time"),
- fill,
- filterSql: filter.sql,
- });
- } else {
- const useDayBucket = bucket === "day" || bucket === "week" || bucket === "month" || bucket === "year";
+ const filtersPresent = hasLiteFilters(req.query.filters);
- query = useDayBucket
- ? buildDayBucketQuery({
- fn,
- tz,
- sessionTime: getLiteTimeStatement(req.query, "session_hour"),
- fill,
- })
- : buildHourBucketQuery({
- fn,
- tz,
- overviewTime: getLiteTimeStatement(req.query, "event_hour"),
- sessionsTime: getLiteTimeStatement(req.query, "start_time"),
- fill,
- });
- }
+ let query: string;
+ if (filtersPresent) {
+ // Session-column filters stay on sessions_mv_target; anything else falls
+ // back to the raw-events query.
+ const filter = getLiteSessionFilter(req.query.filters);
+ if (!filter.supported) {
+ return getOverviewBucketed(req, res);
+ }
+ query = buildSessionMvFilteredQuery({
+ fn,
+ tz,
+ sessionsTime: getLiteTimeStatement(req.query, "start_time"),
+ fill,
+ filterSql: filter.sql,
+ });
+ } else {
+ const useDayBucket = bucket === "day" || bucket === "week" || bucket === "month" || bucket === "year";
- try {
- const result = await clickhouse.query({
+ query = useDayBucket
+ ? buildDayBucketQuery({
+ fn,
+ tz,
+ sessionTime: getLiteTimeStatement(req.query, "session_hour"),
+ fill,
+ })
+ : buildHourBucketQuery({
+ fn,
+ tz,
+ overviewTime: getLiteTimeStatement(req.query, "event_hour"),
+ sessionsTime: getLiteTimeStatement(req.query, "start_time"),
+ fill,
+ });
+ }
+
+ const data = await runAnalyticsQuery<GetOverviewBucketedLiteResponse[number]>({
query,
- format: "JSONEachRow",
- query_params: { siteId: site },
+ params: { siteId: site },
});
- const data = await processResults<GetOverviewBucketedLiteResponse[number]>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching lite bucketed overview:", error);
- return res.status(500).send({ error: "Failed to fetch overview" });
}
-}
+);
diff --git a/server/src/api/analytics/lite/getOverviewLite.ts b/server/src/api/analytics/lite/getOverviewLite.ts
index 51eaa7a0a..77104757e 100644
--- a/server/src/api/analytics/lite/getOverviewLite.ts
+++ b/server/src/api/analytics/lite/getOverviewLite.ts
@@ -1,8 +1,7 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getOverview } from "../getOverview.js";
-import { processResults } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import { getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters } from "./utils.js";
type GetOverviewLiteResponse = {
@@ -14,29 +13,19 @@ type GetOverviewLiteResponse = {
session_duration: number;
};
-export async function getOverviewLite(
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: FilterParams;
- }>,
- res: FastifyReply
-) {
- const site = Number(req.params.siteId);
- const filtersPresent = hasLiteFilters(req.query.filters);
-
- // Filtered queries read sessions_mv_target (one row per session) so any
- // session-column filter — country, device_type, etc. — stays on the MVs.
- // Filters that touch columns the session rollup doesn't carry (pathname,
- // utm, …) fall back to the raw-events query.
- let query: string;
- if (filtersPresent) {
- const filter = getLiteSessionFilter(req.query.filters);
- if (!filter.supported) {
- return getOverview(req, res);
- }
+interface GetOverviewLiteRequest {
+ Params: { siteId: string };
+ Querystring: FilterParams;
+}
- const sessionsTime = getLiteTimeStatement(req.query, "start_time");
- query = `
+// Filtered queries read sessions_mv_target (one row per session) so any
+// session-column filter — country, device_type, etc. — stays on the MVs.
+// Pass `filterSql: null` for the unfiltered single-read of the refreshable
+// session_hourly_mv_target.
+export const buildOverviewLiteQuery = (query: GetOverviewLiteRequest["Querystring"], filterSql: string | null) => {
+ if (filterSql !== null) {
+ const sessionsTime = getLiteTimeStatement(query, "start_time");
+ return `
SELECT
sessions,
pageviews,
@@ -61,23 +50,24 @@ export async function getOverviewLite(
FROM sessions_mv_target
WHERE site_id = {siteId:Int32}
${sessionsTime}
- ${filter.sql}
+ ${filterSql}
GROUP BY session_id
)
)
`;
- } else {
- const timeStatement = getLiteTimeStatement(req.query, "session_hour");
+ }
+
+ const timeStatement = getLiteTimeStatement(query, "session_hour");
- // Single read of the refreshable session_hourly_mv_target — ~720 rows/month
- // per site instead of millions of session rows. All 6 metrics derive from
- // pre-computed sums and one HLL state.
- //
- // Aggregations run in the inner subquery; divisions compose plain values in
- // the outer SELECT. Don't alias `sum(sessions) AS sessions` at this level —
- // ClickHouse will resolve the column name to the aggregate and reject as
- // nested aggregation.
- query = `
+ // Single read of the refreshable session_hourly_mv_target — ~720 rows/month
+ // per site instead of millions of session rows. All 6 metrics derive from
+ // pre-computed sums and one HLL state.
+ //
+ // Aggregations run in the inner subquery; divisions compose plain values in
+ // the outer SELECT. Don't alias `sum(sessions) AS sessions` at this level —
+ // ClickHouse will resolve the column name to the aggregate and reject as
+ // nested aggregation.
+ return `
SELECT
sessions,
pageviews,
@@ -97,18 +87,32 @@ export async function getOverviewLite(
${timeStatement}
)
`;
- }
+};
+
+export const getOverviewLite = analyticsRoute<GetOverviewLiteRequest>(
+ "overview",
+ async (req: FastifyRequest<GetOverviewLiteRequest>, res: FastifyReply) => {
+ const site = Number(req.params.siteId);
+ const filtersPresent = hasLiteFilters(req.query.filters);
+
+ // Filters that touch columns the session rollup doesn't carry (pathname,
+ // utm, …) fall back to the raw-events query.
+ let query: string;
+ if (filtersPresent) {
+ const filter = getLiteSessionFilter(req.query.filters);
+ if (!filter.supported) {
+ return getOverview(req, res);
+ }
+ query = buildOverviewLiteQuery(req.query, filter.sql);
+ } else {
+ query = buildOverviewLiteQuery(req.query, null);
+ }
- try {
- const result = await clickhouse.query({
+ const data = await runAnalyticsQuery<GetOverviewLiteResponse>({
query,
- format: "JSONEachRow",
- query_params: { siteId: site },
+ params: { siteId: site },
});
- const data = await processResults<GetOverviewLiteResponse>(result);
+
return res.send({ data: data[0] });
- } catch (error) {
- console.error("Error fetching lite overview:", error);
- return res.status(500).send({ error: "Failed to fetch overview" });
}
-}
+);
diff --git a/server/src/api/analytics/performance/getPerformanceByDimension.ts b/server/src/api/analytics/performance/getPerformanceByDimension.ts
index bb2a47a2c..9086464cc 100644
--- a/server/src/api/analytics/performance/getPerformanceByDimension.ts
+++ b/server/src/api/analytics/performance/getPerformanceByDimension.ts
@@ -1,8 +1,8 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "../utils/analyticsQuery.js";
interface GetPerformanceByDimensionRequest {
Params: {
@@ -53,9 +53,12 @@ type GetPerformanceByDimensionPaginatedResponse = {
totalCount: number;
};
-const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isCountQuery: boolean = false) => {
- const queryParams = request.query;
- const { filters, limit, page, sort_by: sortBy, sort_order: sortOrder, dimension } = queryParams;
+export const buildPerformanceByDimensionQuery = (
+ query: GetPerformanceByDimensionRequest["Querystring"],
+ siteId: number,
+ isCountQuery: boolean = false
+) => {
+ const { filters, sort_by: sortBy, sort_order: sortOrder, dimension } = query;
// Validate dimension
const validDimensions = ["pathname", "country", "device_type", "browser", "operating_system", "region"];
@@ -64,27 +67,10 @@ const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isC
throw new Error(`Invalid dimension: ${dimension}`);
}
- const timeStatement = getTimeStatement(request.query);
- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
- let validatedLimit: number | null = null;
- if (!isCountQuery && limit !== undefined) {
- const parsedLimit = parseInt(String(limit), 10);
- if (!isNaN(parsedLimit) && parsedLimit > 0) {
- validatedLimit = parsedLimit;
- }
- }
- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 100";
-
- let validatedOffset: number | null = null;
- if (!isCountQuery && page !== undefined) {
- const parsedPage = parseInt(String(page), 10);
- if (!isNaN(parsedPage) && parsedPage >= 1) {
- const pageOffset = (parsedPage - 1) * (validatedLimit || 100);
- validatedOffset = pageOffset;
- }
- }
- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";
+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);
// Handle sorting
const validSortColumns = [
@@ -206,48 +192,17 @@ const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isC
`;
};
-export async function getPerformanceByDimension(
- req: FastifyRequest<GetPerformanceByDimensionRequest>,
- res: FastifyReply
-) {
- const { page, dimension } = req.query;
- const site = req.params.siteId;
-
- const isPaginatedRequest = page !== undefined;
+export const getPerformanceByDimension = analyticsRoute<GetPerformanceByDimensionRequest>(
+ "performance by dimension",
+ async (req: FastifyRequest<GetPerformanceByDimensionRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const params = { siteId };
- try {
- const dataQuery = getQuery(req, false);
- const countQuery = getQuery(req, true);
+ const result = await runPaginatedQuery<PerformanceByDimensionItem>(
+ { query: buildPerformanceByDimensionQuery(req.query, siteId, false), params },
+ { query: buildPerformanceByDimensionQuery(req.query, siteId, true), params }
+ );
- // Run both queries in parallel
- const [dataResult, countResult] = await Promise.all([
- clickhouse.query({
- query: dataQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- }),
- clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- }),
- ]);
-
- const items = await processResults<PerformanceByDimensionItem>(dataResult);
- const countData = await processResults<{ totalCount: number }>(countResult);
- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;
-
- return res.send({ data: { data: items, totalCount } });
- } catch (error) {
- console.error("Error fetching performance by dimension:", error);
- console.error("Failed dataQuery for dimension:", dimension);
- if (isPaginatedRequest) {
- console.error("Failed countQuery for dimension:", dimension);
- }
- return res.status(500).send({ error: "Failed to fetch performance by dimension" });
+ return res.send({ data: result });
}
-}
+);
diff --git a/server/src/api/analytics/performance/getPerformanceOverview.ts b/server/src/api/analytics/performance/getPerformanceOverview.ts
index 6375eb47f..875c8f15c 100644
--- a/server/src/api/analytics/performance/getPerformanceOverview.ts
+++ b/server/src/api/analytics/performance/getPerformanceOverview.ts
@@ -1,13 +1,13 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { PerformanceOverviewMetrics } from "../types.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
-const getQuery = (params: FilterParams, siteId: number) => {
- const timeStatement = getTimeStatement(params);
- const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);
+export const buildPerformanceOverviewQuery = (query: FilterParams, siteId: number) => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
return `SELECT
quantile(0.5)(lcp) AS lcp_p50,
@@ -46,24 +46,16 @@ export interface PerformanceOverviewRequest {
Querystring: FilterParams;
}
-export async function getPerformanceOverview(req: FastifyRequest<PerformanceOverviewRequest>, res: FastifyReply) {
- const site = req.params.siteId;
+export const getPerformanceOverview = analyticsRoute<PerformanceOverviewRequest>(
+ "performance overview",
+ async (req: FastifyRequest<PerformanceOverviewRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
- const query = getQuery(req.query, Number(site));
-
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ const data = await runAnalyticsQuery<PerformanceOverviewMetrics>({
+ query: buildPerformanceOverviewQuery(req.query, siteId),
+ params: { siteId },
});
- const data = await processResults<PerformanceOverviewMetrics>(result);
return res.send({ data: data[0] });
- } catch (error) {
- console.error("Error fetching performance overview:", error);
- return res.status(500).send({ error: "Failed to fetch performance overview" });
}
-}
+);
diff --git a/server/src/api/analytics/performance/getPerformanceTimeSeries.ts b/server/src/api/analytics/performance/getPerformanceTimeSeries.ts
index 0b845eac2..29f36fc00 100644
--- a/server/src/api/analytics/performance/getPerformanceTimeSeries.ts
+++ b/server/src/api/analytics/performance/getPerformanceTimeSeries.ts
@@ -1,11 +1,11 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults, TimeBucketToFn, bucketIntervalMap } from "../utils/utils.js";
+import { getTimeStatement, TimeBucketToFn, bucketIntervalMap } from "../utils/utils.js";
import SqlString from "sqlstring";
import { validateTimeStatementFillParams } from "../utils/query-validation.js";
import { TimeBucket, PerformanceTimeSeriesPoint } from "../types.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);
@@ -81,7 +81,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {
return "";
}
-const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {
+export const buildPerformanceTimeSeriesQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {
const {
start_date,
end_date,
@@ -134,34 +134,25 @@ GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, buck
return query;
};
-export async function getPerformanceTimeSeries(
- req: FastifyRequest<{
- Params: {
- siteId: string;
- };
- Querystring: FilterParams<{
- bucket: TimeBucket;
- }>;
- }>,
- res: FastifyReply
-) {
- const site = req.params.siteId;
+interface GetPerformanceTimeSeriesRequest {
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{
+ bucket: TimeBucket;
+ }>;
+}
- const query = getQuery(req.query, Number(site));
+export const getPerformanceTimeSeries = analyticsRoute<GetPerformanceTimeSeriesRequest>(
+ "performance time series",
+ async (req: FastifyRequest<GetPerformanceTimeSeriesRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
+ const data = await runAnalyticsQuery<PerformanceTimeSeriesPoint>({
+ query: buildPerformanceTimeSeriesQuery(req.query, siteId),
+ params: { siteId },
});
- const data = await processResults<PerformanceTimeSeriesPoint>(result);
return res.send({ data });
- } catch (error) {
- console.error("Error fetching performance time series:", error);
- return res.status(500).send({ error: "Failed to fetch performance time series" });
}
-}
+);
diff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.ts
index 08d532873..71946b3d0 100644
--- a/server/src/api/analytics/sessions/getSession.ts
+++ b/server/src/api/analytics/sessions/getSession.ts
@@ -1,7 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
-import { processResults } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export interface SessionDetails {
session_id: string;
@@ -64,11 +63,8 @@ export interface GetSessionRequest {
};
}
-export async function getSession(req: FastifyRequest<GetSessionRequest>, res: FastifyReply) {
- const { sessionId, siteId } = req.params;
- const limit = req.query.limit ? parseInt(req.query.limit) : 100;
- const offset = req.query.offset ? parseInt(req.query.offset) : 0;
- const minutes = req.query.minutes ? parseInt(req.query.minutes) : undefined;
+export const buildSessionQueries = (query: GetSessionRequest["Querystring"]) => {
+ const minutes = query.minutes ? parseInt(query.minutes) : undefined;
// Add time filter if minutes is provided
const timeFilter = minutes ? `timestamp > now() - interval ${minutes} minute` : "";
@@ -76,9 +72,8 @@ export async function getSession(req: FastifyRequest<GetSessionRequest>, res: Fa
// Add the WHERE clause connector if timeFilter exists
const timeFilterWithConnector = timeFilter ? `AND ${timeFilter}` : "";
- try {
- // 1. First query: Get session data derived from events
- const sessionQuery = `
+ // 1. First query: Get session data derived from events
+ const sessionQuery = `
SELECT
session_id,
any(user_id) as user_id,
@@ -104,7 +99,7 @@ SELECT
argMaxIf(pathname, timestamp, type = 'pageview') as exit_page,
any(ip) AS ip
FROM events
-WHERE
+WHERE
site_id = {siteId:Int32}
AND session_id = {sessionId:String}
${timeFilterWithConnector}
@@ -112,8 +107,8 @@ GROUP BY session_id
LIMIT 1
`;
- // 2. Query to get total count of pageviews
- const countQuery = `
+ // 2. Query to get total count of pageviews
+ const countQuery = `
SELECT
COUNT(*) as total
FROM events
@@ -124,8 +119,8 @@ WHERE
${timeFilterWithConnector}
`;
- // 3. Query to get paginated pageviews
- const eventsQuery = `
+ // 3. Query to get paginated pageviews
+ const eventsQuery = `
SELECT
timestamp,
pathname,
@@ -147,28 +142,37 @@ LIMIT {limit:Int32}
OFFSET {offset:Int32}
`;
+ return { sessionQuery, countQuery, eventsQuery };
+};
+
+export const getSession = analyticsRoute<GetSessionRequest>(
+ "session data",
+ async (req: FastifyRequest<GetSessionRequest>, res: FastifyReply) => {
+ const { sessionId, siteId } = req.params;
+ const limit = req.query.limit ? parseInt(req.query.limit) : 100;
+ const offset = req.query.offset ? parseInt(req.query.offset) : 0;
+
+ const { sessionQuery, countQuery, eventsQuery } = buildSessionQueries(req.query);
+
// Execute queries in parallel
- const [sessionResultSettled, countResultSettled, eventsResultSettled] = await Promise.allSettled([
- clickhouse.query({
+ const [sessionData, countData, eventsData] = await Promise.all([
+ runAnalyticsQuery<SessionDetails>({
query: sessionQuery,
- format: "JSONEachRow",
- query_params: {
+ params: {
siteId: Number(siteId),
sessionId,
},
}),
- clickhouse.query({
+ runAnalyticsQuery<{ total: number }>({
query: countQuery,
- format: "JSONEachRow",
- query_params: {
+ params: {
siteId: Number(siteId),
sessionId,
},
}),
- clickhouse.query({
+ runAnalyticsQuery<Event>({
query: eventsQuery,
- format: "JSONEachRow",
- query_params: {
+ params: {
siteId: Number(siteId),
sessionId,
limit,
@@ -177,25 +181,6 @@ OFFSET {offset:Int32}
}),
]);
- // Check if queries were successful
- if (sessionResultSettled.status === "rejected") {
- throw sessionResultSettled.reason;
- }
- if (countResultSettled.status === "rejected") {
- throw countResultSettled.reason;
- }
- if (eventsResultSettled.status === "rejected") {
- throw eventsResultSettled.reason;
- }
-
- const sessionResult = sessionResultSettled.value;
- const countResult = countResultSettled.value;
- const eventsResult = eventsResultSettled.value;
-
- const sessionData = await processResults<SessionDetails>(sessionResult);
- const countData = await processResults<{ total: number }>(countResult);
- const eventsData = await processResults<Event>(eventsResult);
-
if (!sessionData || sessionData.length === 0) {
return res.status(404).send({ error: "Session not found" });
}
@@ -213,8 +198,5 @@ OFFSET {offset:Int32}
};
return res.send({ data: response });
- } catch (error) {
- console.error("Error fetching session data:", error);
- return res.status(500).send({ error: "Failed to fetch session data" });
}
-}
+);
diff --git a/server/src/api/analytics/sessions/getSessionLocations.ts b/server/src/api/analytics/sessions/getSessionLocations.ts
index 459ecb3a5..60f27cc83 100644
--- a/server/src/api/analytics/sessions/getSessionLocations.ts
+++ b/server/src/api/analytics/sessions/getSessionLocations.ts
@@ -1,25 +1,21 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
-export async function getSessionLocations(
- req: FastifyRequest<{
- Params: {
- siteId: string;
- };
- Querystring: FilterParams<{}>;
- }>,
- res: FastifyReply
-) {
- const { siteId } = req.params;
+export interface GetSessionLocationsRequest {
+ Params: {
+ siteId: string;
+ };
+ Querystring: FilterParams<{}>;
+}
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(req.query.filters, Number(siteId), timeStatement);
+export const buildSessionLocationsQuery = (query: GetSessionLocationsRequest["Querystring"], siteId: number) => {
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
- const result = await clickhouse.query({
- query: `
+ return `
WITH stuff AS (
SELECT
session_id,
@@ -48,19 +44,26 @@ GROUP BY
lat,
lon,
city,
- country`,
- query_params: {
- site: siteId,
- },
- format: "JSONEachRow",
- });
+ country`;
+};
- const data = await processResults<{
- lat: number;
- lon: number;
- count: number;
- city: string;
- }>(result);
+export const getSessionLocations = analyticsRoute<GetSessionLocationsRequest>(
+ "session locations",
+ async (req: FastifyRequest<GetSessionLocationsRequest>, res: FastifyReply) => {
+ const { siteId } = req.params;
- return res.status(200).send({ data });
-}
+ const data = await runAnalyticsQuery<{
+ lat: number;
+ lon: number;
+ count: number;
+ city: string;
+ }>({
+ query: buildSessionLocationsQuery(req.query, Number(siteId)),
+ params: {
+ site: siteId,
+ },
+ });
+
+ return res.status(200).send({ data });
+ }
+);
diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.ts
index bd078fffb..89d5772a6 100644
--- a/server/src/api/analytics/sessions/getSessions.ts
+++ b/server/src/api/analytics/sessions/getSessions.ts
@@ -1,9 +1,9 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery, QuerySpec } from "../utils/analyticsQuery.js";
export type GetSessionsResponse = {
session_id: string;
@@ -76,7 +76,7 @@ const SESSION_FIELD_MAPPINGS = {
"url_parameters['utm_content']": "utm_content",
};
-export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res: FastifyReply) {
+export const buildSessionsQuery = (query: GetSessionsRequest["Querystring"], siteId: number): QuerySpec => {
const {
filters,
page = 1,
@@ -87,26 +87,25 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:
min_pageviews: minPageviewsStr,
min_events: minEventsStr,
min_duration: minDurationStr,
- } = req.query;
- const site = req.params.siteId;
+ } = query;
const filterIdentified = identifiedOnly === "true";
const minPageviews = minPageviewsStr ? parseInt(minPageviewsStr, 10) : undefined;
const minEvents = minEventsStr ? parseInt(minEventsStr, 10) : undefined;
const minDuration = minDurationStr ? parseInt(minDurationStr, 10) : undefined;
- const timeStatement = getTimeStatement(req.query);
+ const timeStatement = getTimeStatement(query);
// Use composable filter options:
// - sessionLevelParams: per-event fields filter at session level (finds sessions
// containing a matching event) — required for any parameter the aggregated CTE
// below doesn't project, otherwise the outer WHERE hits an unknown identifier
// - fieldMappings: CTE extracts UTM params as separate columns, so we need to map the field names
- const filterStatement = getFilterStatement(filters, Number(site), timeStatement, {
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement, {
sessionLevelParams: ["event_name", "pathname", "page_title", "querystring", "channel"],
fieldMappings: SESSION_FIELD_MAPPINGS,
});
- const query = `
+ const querySQL = `
WITH AggregatedSessions AS (
SELECT
session_id,
@@ -179,31 +178,33 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:
LIMIT {limit:Int32} OFFSET {offset:Int32}
`;
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- user_id: userId,
- session_id: sessionId,
- limit: limit || 100,
- offset: (page - 1) * (limit || 100),
- minPageviews: minPageviews ?? 0,
- minEvents: minEvents ?? 0,
- minDuration: minDuration ?? 0,
- },
- });
+ return {
+ query: querySQL,
+ params: {
+ siteId,
+ user_id: userId,
+ session_id: sessionId,
+ limit: limit || 100,
+ offset: (page - 1) * (limit || 100),
+ minPageviews: minPageviews ?? 0,
+ minEvents: minEvents ?? 0,
+ minDuration: minDuration ?? 0,
+ },
+ };
+};
- const data = await processResults<Omit<GetSessionsResponse[number], "traits">>(result);
+export const getSessions = analyticsRoute<GetSessionsRequest>(
+ "sessions",
+ async (req: FastifyRequest<GetSessionsRequest>, res: FastifyReply) => {
+ const site = req.params.siteId;
+
+ const data = await runAnalyticsQuery<Omit<GetSessionsResponse[number], "traits">>(
+ buildSessionsQuery(req.query, Number(site))
+ );
// Enrich with traits from Postgres
const dataWithTraits = await enrichWithTraits(data, Number(site));
return res.send({ data: dataWithTraits });
- } catch (error) {
- console.error("Generated Query:", query);
- console.error("Error fetching sessions:", error);
- return res.status(500).send({ error: "Failed to fetch sessions" });
}
-}
+);
diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.ts
index 51941930b..d36afe7f5 100644
--- a/server/src/api/analytics/users/getUserInfo.ts
+++ b/server/src/api/analytics/users/getUserInfo.ts
@@ -1,12 +1,12 @@
import { FilterParams } from "@rybbit/shared";
import { FastifyReply, FastifyRequest } from "fastify";
import { eq, and } from "drizzle-orm";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { db } from "../../../db/postgres/postgres.js";
import { userProfiles, userAliases } from "../../../db/postgres/schema.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
-import { getTimeStatement, processResults } from "../utils/utils.js";
+import { getTimeStatement } from "../utils/utils.js";
+import { runAnalyticsQuery } from "../utils/analyticsQuery.js";
interface UserPageviewData {
sessions: number;
@@ -84,25 +84,11 @@ export interface UserInfoResponse {
};
}
-export async function getUserInfo(
- req: FastifyRequest<{
- Params: {
- siteId: string;
- userId: string;
- };
- Querystring: FilterParams;
- }>,
- res: FastifyReply
-) {
- const { userId, siteId } = req.params;
- const { filters } = req.query;
-
- const numericSiteId = Number(siteId);
-
+export const buildUserInfoQueries = (query: FilterParams, siteId: number) => {
// Optional time range + dimension filters; both empty when the page is on
// all-time with no filters, which keeps the original full-history behavior.
- const timeStatement = getTimeStatement(req.query);
- const filterStatement = getFilterStatement(filters, numericSiteId, timeStatement);
+ const timeStatement = getTimeStatement(query);
+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);
// Filters run in a subquery below each aggregation: the aggregate SELECTs
// alias argMax(...) to the same names as raw columns (browser_version, …),
@@ -118,10 +104,7 @@ export async function getUserInfo(
${filterStatement}
) AS events`;
- try {
- const [queryResult, vitalsResult, locationsResult, devicesResult, profileResult, aliasesResult] = await Promise.all([
- clickhouse.query({
- query: `
+ const sessionsQuery = `
WITH sessions AS (
SELECT
session_id,
@@ -190,18 +173,12 @@ export async function getUserInfo(
argMaxIf(user_timezone, session_end, user_timezone != '') AS timezone
FROM
sessions
- `,
- query_params: {
- userId,
- site: siteId,
- },
- format: "JSONEachRow",
- }),
- // p75 Web Vitals across every performance event this user produced.
- // Separate query: the sessions CTE collapses rows per session, which
- // would turn an event-level quantile into a quantile of session picks.
- clickhouse.query({
- query: `
+ `;
+
+ // p75 Web Vitals across every performance event this user produced.
+ // Separate query: the sessions CTE collapses rows per session, which
+ // would turn an event-level quantile into a quantile of session picks.
+ const vitalsQuery = `
SELECT
quantile(0.75)(lcp) AS lcp_p75,
quantile(0.75)(cls) AS cls_p75,
@@ -211,17 +188,11 @@ export async function getUserInfo(
COUNT(*) AS performance_events
FROM ${scopedEvents}
WHERE type = 'performance'
- `,
- query_params: {
- userId,
- site: siteId,
- },
- format: "JSONEachRow",
- }),
- // Every location this user was seen in, by session share. A session that
- // moves between cities counts once per city, so shares are approximate.
- clickhouse.query({
- query: `
+ `;
+
+ // Every location this user was seen in, by session share. A session that
+ // moves between cities counts once per city, so shares are approximate.
+ const locationsQuery = `
SELECT
country,
region,
@@ -235,18 +206,12 @@ export async function getUserInfo(
ORDER BY
sessions DESC, last_seen DESC
LIMIT 20
- `,
- query_params: {
- userId,
- site: siteId,
- },
- format: "JSONEachRow",
- }),
- // Every device this user was seen on. Grouped without versions so a
- // browser update doesn't split one physical device into many rows;
- // versions and screen are argMax'd to the latest sighting instead.
- clickhouse.query({
- query: `
+ `;
+
+ // Every device this user was seen on. Grouped without versions so a
+ // browser update doesn't split one physical device into many rows;
+ // versions and screen are argMax'd to the latest sighting instead.
+ const devicesQuery = `
SELECT
device_type,
browser,
@@ -264,13 +229,38 @@ export async function getUserInfo(
ORDER BY
sessions DESC, last_seen DESC
LIMIT 20
- `,
- query_params: {
- userId,
- site: siteId,
- },
- format: "JSONEachRow",
- }),
+ `;
+
+ return { sessionsQuery, vitalsQuery, locationsQuery, devicesQuery };
+};
+
+export async function getUserInfo(
+ req: FastifyRequest<{
+ Params: {
+ siteId: string;
+ userId: string;
+ };
+ Querystring: FilterParams;
+ }>,
+ res: FastifyReply
+) {
+ const { userId, siteId } = req.params;
+
+ const numericSiteId = Number(siteId);
+
+ const { sessionsQuery, vitalsQuery, locationsQuery, devicesQuery } = buildUserInfoQueries(req.query, numericSiteId);
+
+ const chParams = {
+ userId,
+ site: siteId,
+ };
+
+ try {
+ const [data, vitalsData, locations, devices, profileResult, aliasesResult] = await Promise.all([
+ runAnalyticsQuery<UserPageviewData>({ query: sessionsQuery, params: chParams }),
+ runAnalyticsQuery<UserVitalsData>({ query: vitalsQuery, params: chParams }),
+ runAnalyticsQuery<UserLocationBreakdown>({ query: locationsQuery, params: chParams }),
+ runAnalyticsQuery<UserDeviceBreakdown>({ query: devicesQuery, params: chParams }),
// Get user profile traits from Postgres
db
.select()
@@ -287,11 +277,6 @@ export async function getUserInfo(
.where(and(eq(userAliases.siteId, numericSiteId), eq(userAliases.userId, userId))),
]);
- const data = await processResults<UserPageviewData>(queryResult);
- const vitalsData = await processResults<UserVitalsData>(vitalsResult);
- const locations = await processResults<UserLocationBreakdown>(locationsResult);
- const devices = await processResults<UserDeviceBreakdown>(devicesResult);
-
// If no data found for user
if (data.length === 0) {
return res.status(404).send({
diff --git a/server/src/api/analytics/users/getUserSessionCount.ts b/server/src/api/analytics/users/getUserSessionCount.ts
index 5e90f2e85..e531d1270 100644
--- a/server/src/api/analytics/users/getUserSessionCount.ts
+++ b/server/src/api/analytics/users/getUserSessionCount.ts
@@ -1,7 +1,6 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { getFilterStatement } from "../utils/getFilterStatement.js";
-import { processResults } from "../utils/utils.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
import SqlString from "sqlstring";
export interface GetUserSessionCountRequest {
@@ -20,19 +19,14 @@ export type GetUserSessionCountResponse = {
sessions: number;
}[];
-export async function getUserSessionCount(req: FastifyRequest<GetUserSessionCountRequest>, res: FastifyReply) {
- const { siteId } = req.params;
- const { user_id: userId, time_zone: timeZone = "UTC", filters } = req.query;
-
- if (!userId) {
- return res.status(400).send({ error: "user_id is required" });
- }
+export const buildUserSessionCountQuery = (query: GetUserSessionCountRequest["Querystring"], siteId: number) => {
+ const { time_zone: timeZone = "UTC", filters } = query;
// The calendar spans the user's full history, so dimension filters apply
// but no time range does.
- const filterStatement = getFilterStatement(filters ?? "", Number(siteId));
+ const filterStatement = getFilterStatement(filters ?? "", siteId);
- const query = `
+ return `
SELECT
toDate(timestamp, ${SqlString.escape(timeZone)}) as date,
count(DISTINCT session_id) as sessions
@@ -44,24 +38,28 @@ export async function getUserSessionCount(req: FastifyRequest<GetUserSessionCoun
GROUP BY date
ORDER BY date ASC
`;
+};
+
+export const getUserSessionCount = analyticsRoute<GetUserSessionCountRequest>(
+ "user session count",
+ async (req: FastifyRequest<GetUserSessionCountRequest>, res: FastifyReply) => {
+ const { siteId } = req.params;
+ const { user_id: userId } = req.query;
- try {
- const result = await clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ if (!userId) {
+ return res.status(400).send({ error: "user_id is required" });
+ }
+
+ const data = await runAnalyticsQuery<GetUserSessionCountResponse[number]>({
+ query: buildUserSessionCountQuery(req.query, Number(siteId)),
+ params: {
siteId: Number(siteId),
userId,
},
});
- const data = await processResults<GetUserSessionCountResponse[number]>(result);
-
return res.send({
data,
});
- } catch (error) {
- console.error("Error fetching user session count:", error);
- return res.status(500).send({ error: "Failed to fetch user session count" });
}
-}
+);
diff --git a/server/src/api/analytics/users/getUserTraits.ts b/server/src/api/analytics/users/getUserTraits.ts
index 008506046..4ee875274 100644
--- a/server/src/api/analytics/users/getUserTraits.ts
+++ b/server/src/api/analytics/users/getUserTraits.ts
@@ -1,18 +1,27 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { sql } from "drizzle-orm";
import { db } from "../../../db/postgres/postgres.js";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { processResults } from "../utils/utils.js";
-
-export async function getUserTraitKeys(
- req: FastifyRequest<{
- Params: { siteId: string };
- }>,
- res: FastifyReply
-) {
- const siteId = Number(req.params.siteId);
-
- try {
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
+
+export interface GetUserTraitKeysRequest {
+ Params: { siteId: string };
+}
+
+export interface GetUserTraitValuesRequest {
+ Params: { siteId: string };
+ Querystring: { key: string; limit?: string; offset?: string };
+}
+
+export interface GetUserTraitValueUsersRequest {
+ Params: { siteId: string };
+ Querystring: { key: string; value: string; limit?: string; offset?: string };
+}
+
+export const getUserTraitKeys = analyticsRoute<GetUserTraitKeysRequest>(
+ "user trait keys",
+ async (req: FastifyRequest<GetUserTraitKeysRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+
const result = await db.execute<{ key: string; user_count: number }>(sql`
SELECT key, COUNT(*)::int AS user_count
FROM user_profiles,
@@ -28,30 +37,22 @@ export async function getUserTraitKeys(
userCount: row.user_count,
})),
});
- } catch (error) {
- console.error("Error fetching user trait keys:", error);
- return res.status(500).send({ error: "Failed to fetch user trait keys" });
}
-}
+);
-export async function getUserTraitValues(
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: { key: string; limit?: string; offset?: string };
- }>,
- res: FastifyReply
-) {
- const siteId = Number(req.params.siteId);
- const { key, limit: limitStr = "1000", offset: offsetStr = "0" } = req.query;
-
- if (!key) {
- return res.status(400).send({ error: "key query parameter is required" });
- }
+export const getUserTraitValues = analyticsRoute<GetUserTraitValuesRequest>(
+ "user trait values",
+ async (req: FastifyRequest<GetUserTraitValuesRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const { key, limit: limitStr = "1000", offset: offsetStr = "0" } = req.query;
+
+ if (!key) {
+ return res.status(400).send({ error: "key query parameter is required" });
+ }
- const limit = parseInt(limitStr, 10);
- const offset = parseInt(offsetStr, 10);
+ const limit = parseInt(limitStr, 10);
+ const offset = parseInt(offsetStr, 10);
- try {
const [valuesResult, countResult] = await Promise.all([
db.execute<{ value: string; user_count: number }>(sql`
SELECT traits->>${key} AS value, COUNT(*)::int AS user_count
@@ -78,30 +79,42 @@ export async function getUserTraitValues(
total,
hasMore: offset + limit < total,
});
- } catch (error) {
- console.error("Error fetching user trait values:", error);
- return res.status(500).send({ error: "Failed to fetch user trait values" });
}
-}
+);
-export async function getUserTraitValueUsers(
- req: FastifyRequest<{
- Params: { siteId: string };
- Querystring: { key: string; value: string; limit?: string; offset?: string };
- }>,
- res: FastifyReply
-) {
- const siteId = Number(req.params.siteId);
- const { key, value, limit: limitStr = "50", offset: offsetStr = "0" } = req.query;
-
- if (!key || value === undefined) {
- return res.status(400).send({ error: "key and value query parameters are required" });
- }
+// Session counts + metadata from ClickHouse for a set of identified users.
+// Use events.identified_user_id to avoid conflict with the argMax alias
+export const buildTraitValueUsersQuery = () => `
+ SELECT
+ COALESCE(NULLIF(events.identified_user_id, ''), events.user_id) AS effective_user_id,
+ argMax(events.user_id, timestamp) AS user_id,
+ argMax(events.identified_user_id, timestamp) AS identified_user_id,
+ argMax(country, timestamp) AS country,
+ argMax(region, timestamp) AS region,
+ argMax(city, timestamp) AS city,
+ argMax(browser, timestamp) AS browser,
+ argMax(operating_system, timestamp) AS operating_system,
+ argMax(device_type, timestamp) AS device_type,
+ count(DISTINCT session_id) AS sessions
+ FROM events
+ WHERE site_id = {siteId:Int32}
+ AND events.identified_user_id IN ({userIds:Array(String)})
+ GROUP BY effective_user_id
+ `;
- const limit = parseInt(limitStr, 10);
- const offset = parseInt(offsetStr, 10);
+export const getUserTraitValueUsers = analyticsRoute<GetUserTraitValueUsersRequest>(
+ "trait value users",
+ async (req: FastifyRequest<GetUserTraitValueUsersRequest>, res: FastifyReply) => {
+ const siteId = Number(req.params.siteId);
+ const { key, value, limit: limitStr = "50", offset: offsetStr = "0" } = req.query;
+
+ if (!key || value === undefined) {
+ return res.status(400).send({ error: "key and value query parameters are required" });
+ }
+
+ const limit = parseInt(limitStr, 10);
+ const offset = parseInt(offsetStr, 10);
- try {
// Step 1: Find matching user IDs and their traits from Postgres
const [profilesResult, countResult] = await Promise.all([
db.execute<{ user_id: string; traits: Record<string, unknown> | null }>(sql`
@@ -128,35 +141,7 @@ export async function getUserTraitValueUsers(
const userIds = profiles.map((p: any) => p.user_id);
// Step 2: Get session counts + metadata from ClickHouse
- // Use events.identified_user_id to avoid conflict with the argMax alias
- const chQuery = `
- SELECT
- COALESCE(NULLIF(events.identified_user_id, ''), events.user_id) AS effective_user_id,
- argMax(events.user_id, timestamp) AS user_id,
- argMax(events.identified_user_id, timestamp) AS identified_user_id,
- argMax(country, timestamp) AS country,
- argMax(region, timestamp) AS region,
- argMax(city, timestamp) AS city,
- argMax(browser, timestamp) AS browser,
- argMax(operating_system, timestamp) AS operating_system,
- argMax(device_type, timestamp) AS device_type,
- count(DISTINCT session_id) AS sessions
- FROM events
- WHERE site_id = {siteId:Int32}
- AND events.identified_user_id IN ({userIds:Array(String)})
- GROUP BY effective_user_id
- `;
-
- const chResult = await clickhouse.query({
- query: chQuery,
- format: "JSONEachRow",
- query_params: {
- siteId,
- userIds,
- },
- });
-
- const chData = await processResults<{
+ const chData = await runAnalyticsQuery<{
effective_user_id: string;
user_id: string;
identified_user_id: string;
@@ -167,7 +152,13 @@ export async function getUserTraitValueUsers(
operating_system: string;
device_type: string;
sessions: number;
- }>(chResult);
+ }>({
+ query: buildTraitValueUsersQuery(),
+ params: {
+ siteId,
+ userIds,
+ },
+ });
// Step 3: Build lookup from ClickHouse data keyed by identified_user_id,
// then iterate Postgres profiles so every profile is returned even if
@@ -197,8 +188,5 @@ export async function getUserTraitValueUsers(
total,
hasMore: offset + limit < total,
});
- } catch (error) {
- console.error("Error fetching trait value users:", error);
- return res.status(500).send({ error: "Failed to fetch trait value users" });
}
-}
+);
diff --git a/server/src/api/analytics/users/getUsers.ts b/server/src/api/analytics/users/getUsers.ts
index ecc814f38..25b9841f5 100644
--- a/server/src/api/analytics/users/getUsers.ts
+++ b/server/src/api/analytics/users/getUsers.ts
@@ -1,11 +1,11 @@
import { FastifyReply, FastifyRequest } from "fastify";
import { sql, SQL } from "drizzle-orm";
-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
import { db } from "../../../db/postgres/postgres.js";
-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";
+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";
import { FilterParams } from "@rybbit/shared";
import { getFilterStatement } from "../utils/getFilterStatement.js";
import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";
+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";
export type GetUsersResponse = {
user_id: string; // Device fingerprint
@@ -41,56 +41,21 @@ export interface GetUsersRequest {
}>;
}
-export async function getUsers(req: FastifyRequest<GetUsersRequest>, res: FastifyReply) {
+export const buildUsersQuery = (
+ query: GetUsersRequest["Querystring"],
+ siteId: number,
+ matchingUserIds: string[] | null,
+ isCountQuery: boolean = false
+) => {
const {
filters,
- page = "1",
- page_size: pageSize = "100",
sort_by: sortBy = "last_seen",
sort_order: sortOrder = "desc",
identified_only: identifiedOnly = "false",
- search,
- search_field: searchField = "username",
- } = req.query;
- const site = req.params.siteId;
- let filterIdentified = identifiedOnly === "true";
-
- // Search for matching user IDs in Postgres when search is provided
- const MAX_MATCHING_USER_IDS = 10000;
- let matchingUserIds: string[] | null = null;
- if (search && search.trim()) {
- const searchTerm = `%${search.trim()}%`;
- const siteId = Number(site);
-
- const fieldConditions: Record<string, SQL> = {
- username: sql`traits->>'username' ILIKE ${searchTerm}`,
- name: sql`traits->>'name' ILIKE ${searchTerm}`,
- email: sql`traits->>'email' ILIKE ${searchTerm}`,
- user_id: sql`user_id ILIKE ${searchTerm}`,
- };
- const condition = fieldConditions[searchField] ?? fieldConditions.username;
-
- const searchResult = await db.execute<{ user_id: string }>(sql`
- SELECT user_id FROM user_profiles
- WHERE site_id = ${siteId} AND ${condition}
- LIMIT ${MAX_MATCHING_USER_IDS}
- `);
-
- matchingUserIds = searchResult.map((r) => r.user_id);
- if (matchingUserIds.length === 0) {
- return res.send({
- data: [],
- totalCount: 0,
- page: parseInt(page, 10),
- pageSize: parseInt(pageSize, 10),
- });
- }
- filterIdentified = true;
- }
-
- const pageNum = parseInt(page, 10);
- const pageSizeNum = parseInt(pageSize, 10);
- const offset = (pageNum - 1) * pageSizeNum;
+ } = query;
+ // Search results force the identified-only view: matching user IDs come from
+ // Postgres profiles, which only exist for identified users.
+ const filterIdentified = identifiedOnly === "true" || matchingUserIds !== null;
// Validate sort parameters
const validSortFields = ["first_seen", "last_seen", "pageviews", "sessions", "events"];
@@ -98,18 +63,46 @@ export async function getUsers(req: FastifyRequest<GetUsersRequest>, res: Fastif
const actualSortOrder = sortOrder === "asc" ? "ASC" : "DESC";
// Generate filter statement and time statement
- const timeStatement = getTimeStatement(req.query);
+ const timeStatement = getTimeStatement(query);
// Applied against raw events (same placement as the count queries): the
// aggregate doesn't project every filterable column (pathname, querystring,
// utm_*, …), and event-level placement keeps the returned rows consistent
// with totalCount.
- const filterStatement = getFilterStatement(filters, Number(site), timeStatement);
+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);
+
+ // Query to get total count
+ if (isCountQuery) {
+ return filterIdentified
+ ? `
+SELECT count(*) AS total_count
+FROM (
+ SELECT DISTINCT identified_user_id
+ FROM events
+ WHERE
+ site_id = {siteId:Int32}
+ AND identified_user_id != ''
+ ${timeStatement}
+ ${filterStatement}
+ ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}
+)
+`
+ : `
+SELECT
+ count(DISTINCT COALESCE(NULLIF(events.identified_user_id, ''), events.user_id)) AS total_count
+FROM events
+WHERE
+ site_id = {siteId:Int32}
+ ${filterStatement}
+ ${timeStatement}
+ ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}
+ `;
+ }
// Filters must run in a subquery below the aggregation: the aggregate SELECT
// aliases argMax(...) to the same names as the raw columns (country, browser,
// …), and ClickHouse resolves unqualified WHERE references at that level to
// the aliases, throwing ILLEGAL_AGGREGATION.
- const query = `
+ return `
WITH AggregatedUsers AS (
SELECT
-- Group by effective user: identified_user_id for identified users, user_id (device) for anonymous
@@ -156,58 +149,70 @@ ${filterIdentified ? "AND identified_user_id != ''" : ""}
ORDER BY ${actualSortBy} ${actualSortOrder}
LIMIT {limit:Int32} OFFSET {offset:Int32}
`;
+};
+
+export const getUsers = analyticsRoute<GetUsersRequest>(
+ "users",
+ async (req: FastifyRequest<GetUsersRequest>, res: FastifyReply) => {
+ const { page = "1", page_size: pageSize = "100", search, search_field: searchField = "username" } = req.query;
+ const site = req.params.siteId;
+
+ // Search for matching user IDs in Postgres when search is provided
+ const MAX_MATCHING_USER_IDS = 10000;
+ let matchingUserIds: string[] | null = null;
+ if (search && search.trim()) {
+ const searchTerm = `%${search.trim()}%`;
+ const siteId = Number(site);
+
+ const fieldConditions: Record<string, SQL> = {
+ username: sql`traits->>'username' ILIKE ${searchTerm}`,
+ name: sql`traits->>'name' ILIKE ${searchTerm}`,
+ email: sql`traits->>'email' ILIKE ${searchTerm}`,
+ user_id: sql`user_id ILIKE ${searchTerm}`,
+ };
+ const condition = fieldConditions[searchField] ?? fieldConditions.username;
+
+ const searchResult = await db.execute<{ user_id: string }>(sql`
+ SELECT user_id FROM user_profiles
+ WHERE site_id = ${siteId} AND ${condition}
+ LIMIT ${MAX_MATCHING_USER_IDS}
+ `);
+
+ matchingUserIds = searchResult.map((r) => r.user_id);
+ if (matchingUserIds.length === 0) {
+ return res.send({
+ data: [],
+ totalCount: 0,
+ page: parseInt(page, 10),
+ pageSize: parseInt(pageSize, 10),
+ });
+ }
+ }
- // Query to get total count
- const countQuery = filterIdentified
- ? `
-SELECT count(*) AS total_count
-FROM (
- SELECT DISTINCT identified_user_id
- FROM events
- WHERE
- site_id = {siteId:Int32}
- AND identified_user_id != ''
- ${timeStatement}
- ${filterStatement}
- ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}
-)
-`
- : `
-SELECT
- count(DISTINCT COALESCE(NULLIF(events.identified_user_id, ''), events.user_id)) AS total_count
-FROM events
-WHERE
- site_id = {siteId:Int32}
- ${filterStatement}
- ${timeStatement}
- ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}
- `;
+ const pageNum = parseInt(page, 10);
+ const pageSizeNum = parseInt(pageSize, 10);
+ const offset = (pageNum - 1) * pageSizeNum;
- try {
// Execute both queries in parallel
- const [result, countResult] = await Promise.all([
- clickhouse.query({
- query,
- format: "JSONEachRow",
- query_params: {
+ const [data, countData] = await Promise.all([
+ runAnalyticsQuery<Omit<GetUsersResponse[number], "traits">>({
+ query: buildUsersQuery(req.query, Number(site), matchingUserIds, false),
+ params: {
siteId: Number(site),
limit: pageSizeNum,
offset,
...(matchingUserIds ? { matchingUserIds } : {}),
},
}),
- clickhouse.query({
- query: countQuery,
- format: "JSONEachRow",
- query_params: {
+ runAnalyticsQuery<{ total_count: number }>({
+ query: buildUsersQuery(req.query, Number(site), matchingUserIds, true),
+ params: {
siteId: Number(site),
...(matchingUserIds ? { matchingUserIds } : {}),
},
}),
]);
- const data = await processResults<Omit<GetUsersResponse[number], "traits">>(result);
- const countData = await processResults<{ total_count: number }>(countResult);
const totalCount = countData[0]?.total_count || 0;
// Enrich with traits from Postgres
@@ -219,8 +224,5 @@ WHERE
page: pageNum,
pageSize: pageSizeNum,
});
- } catch (error) {
- console.error("Error fetching users:", error);
- return res.status(500).send({ error: "Failed to fetch users" });
}
-}
+);
diff --git a/server/src/api/analytics/utils/analyticsQuery.test.ts b/server/src/api/analytics/utils/analyticsQuery.test.ts
new file mode 100644
index 000000000..42ff966e8
--- /dev/null
+++ b/server/src/api/analytics/utils/analyticsQuery.test.ts
@@ -0,0 +1,161 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../../../db/clickhouse/clickhouse.js", () => ({
+ clickhouse: { query: vi.fn() },
+}));
+vi.mock("../../../db/postgres/postgres.js", () => ({
+ db: {},
+}));
+
+import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
+import {
+ AnalyticsQueryError,
+ analyticsRoute,
+ getPaginationStatements,
+ runAnalyticsQuery,
+ runPaginatedQuery,
+} from "./analyticsQuery.js";
+
+const mockQuery = vi.mocked(clickhouse.query);
+
+const resultSet = (rows: unknown[]) => ({ json: async () => rows }) as any;
+
+const mockRes = () => {
+ const res: any = {};
+ res.status = vi.fn(() => res);
+ res.send = vi.fn(() => res);
+ return res;
+};
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("getPaginationStatements", () => {
+ it("falls back to the default limit", () => {
+ expect(getPaginationStatements({}, 100)).toEqual({ limitStatement: "LIMIT 100", offsetStatement: "" });
+ });
+
+ it("uses a valid explicit limit, numeric or string", () => {
+ expect(getPaginationStatements({ limit: 25 }, 100).limitStatement).toBe("LIMIT 25");
+ expect(getPaginationStatements({ limit: "25" }, 100).limitStatement).toBe("LIMIT 25");
+ });
+
+ it("rejects invalid limits", () => {
+ expect(getPaginationStatements({ limit: "abc" }, 100).limitStatement).toBe("LIMIT 100");
+ expect(getPaginationStatements({ limit: -5 }, 100).limitStatement).toBe("LIMIT 100");
+ expect(getPaginationStatements({ limit: 0 }, 100).limitStatement).toBe("LIMIT 100");
+ });
+
+ it("computes the offset from page and limit", () => {
+ expect(getPaginationStatements({ limit: 25, page: 3 }, 100).offsetStatement).toBe("OFFSET 50");
+ expect(getPaginationStatements({ page: 2 }, 10).offsetStatement).toBe("OFFSET 10");
+ });
+
+ it("emits no OFFSET for page 1 or invalid pages", () => {
+ expect(getPaginationStatements({ page: 1 }, 100).offsetStatement).toBe("");
+ expect(getPaginationStatements({ page: 0 }, 100).offsetStatement).toBe("");
+ expect(getPaginationStatements({ page: "abc" }, 100).offsetStatement).toBe("");
+ });
+
+ it("emits no clauses for count queries", () => {
+ expect(getPaginationStatements({ limit: 25, page: 3 }, 100, true)).toEqual({
+ limitStatement: "",
+ offsetStatement: "",
+ });
+ });
+});
+
+describe("runAnalyticsQuery", () => {
+ it("executes the spec and coerces numeric strings", async () => {
+ mockQuery.mockResolvedValueOnce(resultSet([{ value: "Chrome", count: "42" }]));
+
+ const rows = await runAnalyticsQuery<{ value: string; count: number }>({
+ query: "SELECT 1",
+ params: { siteId: 1 },
+ });
+
+ expect(mockQuery).toHaveBeenCalledWith({
+ query: "SELECT 1",
+ format: "JSONEachRow",
+ query_params: { siteId: 1 },
+ });
+ expect(rows).toEqual([{ value: "Chrome", count: 42 }]);
+ });
+
+ it("wraps failures in AnalyticsQueryError carrying the SQL", async () => {
+ const boom = new Error("boom");
+ mockQuery.mockRejectedValueOnce(boom);
+
+ const promise = runAnalyticsQuery({ query: "SELECT bad" });
+
+ await expect(promise).rejects.toBeInstanceOf(AnalyticsQueryError);
+ await promise.catch((error: AnalyticsQueryError) => {
+ expect(error.original).toBe(boom);
+ expect(error.queries).toEqual(["SELECT bad"]);
+ });
+ });
+});
+
+describe("runPaginatedQuery", () => {
+ it("runs data and count queries and combines them", async () => {
+ mockQuery
+ .mockResolvedValueOnce(resultSet([{ value: "a" }, { value: "b" }]))
+ .mockResolvedValueOnce(resultSet([{ totalCount: "17" }]));
+
+ const result = await runPaginatedQuery({ query: "SELECT data" }, { query: "SELECT count" });
+
+ expect(result).toEqual({ data: [{ value: "a" }, { value: "b" }], totalCount: 17 });
+ });
+
+ it("defaults totalCount to 0 when the count query returns no rows", async () => {
+ mockQuery.mockResolvedValueOnce(resultSet([])).mockResolvedValueOnce(resultSet([]));
+
+ const result = await runPaginatedQuery({ query: "SELECT data" }, { query: "SELECT count" });
+
+ expect(result).toEqual({ data: [], totalCount: 0 });
+ });
+});
+
+describe("analyticsRoute", () => {
+ it("passes successful responses through", async () => {
+ const res = mockRes();
+ const handler = analyticsRoute("things", async (_req, reply) => reply.send({ data: [1] }));
+
+ await handler({} as any, res);
+
+ expect(res.send).toHaveBeenCalledWith({ data: [1] });
+ expect(res.status).not.toHaveBeenCalled();
+ });
+
+ it("maps errors to a 500 with the label", async () => {
+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+ const res = mockRes();
+ const handler = analyticsRoute("things", async () => {
+ throw new Error("boom");
+ });
+
+ await handler({} as any, res);
+
+ expect(res.status).toHaveBeenCalledWith(500);
+ expect(res.send).toHaveBeenCalledWith({ error: "Failed to fetch things" });
+ consoleError.mockRestore();
+ });
+
+ it("derives the label from the request and logs failed SQL", async () => {
+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+ const res = mockRes();
+ const handler = analyticsRoute<{ Querystring: { parameter: string } }>(
+ req => req.query.parameter,
+ async () => {
+ throw new AnalyticsQueryError(new Error("boom"), ["SELECT bad"]);
+ }
+ );
+
+ await handler({ query: { parameter: "browser" } } as any, res);
+
+ expect(res.send).toHaveBeenCalledWith({ error: "Failed to fetch browser" });
+ expect(consoleError).toHaveBeenCalledWith("Failed query:", "SELECT bad");
+ consoleError.mockRestore();
+ });
+});
diff --git a/server/src/api/analytics/utils/analyticsQuery.ts b/server/src/api/analytics/utils/analyticsQuery.ts
new file mode 100644
index 000000000..341b10822
--- /dev/null
+++ b/server/src/api/analytics/utils/analyticsQuery.ts
@@ -0,0 +1,117 @@
+import { FastifyReply, FastifyRequest, RouteGenericInterface } from "fastify";
+import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
+import { processResults } from "./utils.js";
+
+/**
+ * A fully-built ClickHouse query plus its bound parameters. Handlers build
+ * specs with pure functions (testable without Fastify or a live database);
+ * only the executor below ever talks to ClickHouse.
+ */
+export interface QuerySpec {
+ query: string;
+ params?: Record<string, unknown>;
+}
+
+/** Carries the failed SQL so the route wrapper can log it alongside the cause. */
+export class AnalyticsQueryError extends Error {
+ constructor(
+ readonly original: unknown,
+ readonly queries: string[]
+ ) {
+ super("ClickHouse query failed");
+ this.name = "AnalyticsQueryError";
+ }
+}
+
+export async function runAnalyticsQuery<T>(spec: QuerySpec): Promise<T[]> {
+ try {
+ const result = await clickhouse.query({
+ query: spec.query,
+ format: "JSONEachRow",
+ query_params: spec.params,
+ });
+ return await processResults<T>(result);
+ } catch (error) {
+ throw new AnalyticsQueryError(error, [spec.query]);
+ }
+}
+
+export async function runPaginatedQuery<T>(
+ dataSpec: QuerySpec,
+ countSpec: QuerySpec
+): Promise<{ data: T[]; totalCount: number }> {
+ const [data, countRows] = await Promise.all([
+ runAnalyticsQuery<T>(dataSpec),
+ runAnalyticsQuery<{ totalCount: number }>(countSpec),
+ ]);
+ return { data, totalCount: countRows[0]?.totalCount ?? 0 };
+}
+
+/**
+ * LIMIT/OFFSET clauses from untrusted `limit`/`page` query params.
+ * Count queries get no clauses; invalid or missing values fall back to
+ * `defaultLimit`; page 1 (offset 0) emits no OFFSET clause.
+ */
+export function getPaginationStatements(
+ { limit, page }: { limit?: number | string; page?: number | string },
+ defaultLimit: number,
+ isCountQuery = false
+): { limitStatement: string; offsetStatement: string } {
+ if (isCountQuery) {
+ return { limitStatement: "", offsetStatement: "" };
+ }
+
+ let validatedLimit: number | null = null;
+ if (limit !== undefined) {
+ const parsedLimit = parseInt(String(limit), 10);
+ if (!isNaN(parsedLimit) && parsedLimit > 0) {
+ validatedLimit = parsedLimit;
+ }
+ }
+ const limitStatement = `LIMIT ${validatedLimit ?? defaultLimit}`;
+
+ let offsetStatement = "";
+ if (page !== undefined) {
+ const parsedPage = parseInt(String(page), 10);
+ if (!isNaN(parsedPage) && parsedPage >= 1) {
+ const offset = (parsedPage - 1) * (validatedLimit ?? defaultLimit);
+ if (offset > 0) {
+ offsetStatement = `OFFSET ${offset}`;
+ }
+ }
+ }
+
+ return { limitStatement, offsetStatement };
+}
+
+type AnalyticsHandler<R extends RouteGenericInterface> = (
+ req: FastifyRequest<R>,
+ res: FastifyReply
+) => Promise<unknown>;
+
+/**
+ * Wraps a handler with the standard analytics error policy: failures log the
+ * label (and the failed SQL when available) and respond 500 with
+ * `Failed to fetch <label>`. The label may derive from the request.
+ */
+export function analyticsRoute<R extends RouteGenericInterface>(
+ errorLabel: string | ((req: FastifyRequest<R>) => string),
+ handler: AnalyticsHandler<R>
+): AnalyticsHandler<R> {
+ return async (req, res) => {
+ try {
+ return await handler(req, res);
+ } catch (error) {
+ const label = typeof errorLabel === "function" ? errorLabel(req) : errorLabel;
+ if (error instanceof AnalyticsQueryError) {
+ console.error(`Error fetching ${label}:`, error.original);
+ for (const query of error.queries) {
+ console.error("Failed query:", query);
+ }
+ } else {
+ console.error(`Error fetching ${label}:`, error);
+ }
+ return res.status(500).send({ error: `Failed to fetch ${label}` });
+ }
+ };
+}
diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.ts
index 2a08a2c41..23f95f1b3 100644
--- a/server/src/api/analytics/utils/utils.ts
+++ b/server/src/api/analytics/utils/utils.ts
@@ -107,6 +107,7 @@ export async function processResults<T>(results: ResultSet<"JSONEachRow">): Prom
row[key] !== "" &&
row[key] !== true &&
row[key] !== false &&
+ !Array.isArray(row[key]) &&
!isNaN(Number(row[key]))
) {
row[key] = Number(row[key]) as any;