🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor analytics API endpoints to improve query handling and error management
Bill Yang66b49e9925cc
diff --git a/server/src/api/analytics/bots/getBotDimension.ts b/server/src/api/analytics/bots/getBotDimension.tsindex 84dc7ca7b..63f5466d7 100644--- a/server/src/api/analytics/bots/getBotDimension.ts+++ b/server/src/api/analytics/bots/getBotDimension.ts@@ -1,7 +1,7 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "../utils/analyticsQuery.js";import {BOT_DIMENSIONS,type BotDimensionKey,@@ -35,24 +35,17 @@ export interface BotDimensionRequest {}>;}-const parsePositiveInt = (value: unknown, fallback: number) => {- const parsed = parseInt(String(value), 10);- return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;-};--const getQuery = (request: FastifyRequest<BotDimensionRequest>, isCountQuery = false) => {- const { dimension, limit, page } = request.query;+export const buildBotDimensionQuery = (query: BotDimensionRequest["Querystring"], isCountQuery = false) => {+ const { dimension } = query;if (!BOT_DIMENSIONS.has(dimension)) {throw new Error(`Unsupported bot dimension: ${dimension}`);}const expression = getBotSqlParam(dimension);- const timeStatement = getTimeStatement(request.query);- const filterStatement = getBotFilterStatement(request.query.filters);- const layerStatement = getBotLayerStatement(request.query.layer);- const validatedLimit = parsePositiveInt(limit, 100);- const validatedPage = parsePositiveInt(page, 1);- const offset = (validatedPage - 1) * validatedLimit;+ const timeStatement = getTimeStatement(query);+ const filterStatement = getBotFilterStatement(query.filters);+ const layerStatement = getBotLayerStatement(query.layer);+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);const groupedQuery = `SELECT@@ -78,40 +71,21 @@ const getQuery = (request: FastifyRequest<BotDimensionRequest>, isCountQuery = freturn `${groupedQuery}ORDER BY count DESC- LIMIT ${validatedLimit}- OFFSET ${offset}+ ${limitStatement}+ ${offsetStatement}`;};-export async function getBotDimension(req: FastifyRequest<BotDimensionRequest>, res: FastifyReply) {- try {- const [dataResult, countResult] = await Promise.all([- clickhouse.query({- query: getQuery(req),- format: "JSONEachRow",- query_params: {- siteId: Number(req.params.siteId),- },- }),- clickhouse.query({- query: getQuery(req, true),- format: "JSONEachRow",- query_params: {- siteId: Number(req.params.siteId),- },- }),- ]);+export const getBotDimension = analyticsRoute<BotDimensionRequest>(+ "bot dimension",+ async (req: FastifyRequest<BotDimensionRequest>, res: FastifyReply) => {+ const params = { siteId: Number(req.params.siteId) };- const data = await processResults<BotDimensionItem>(dataResult);- const countData = await processResults<{ totalCount: number }>(countResult);- const response: BotDimensionResponse = {- data,- totalCount: countData[0]?.totalCount ?? 0,- };+ const response: BotDimensionResponse = await runPaginatedQuery<BotDimensionItem>(+ { query: buildBotDimensionQuery(req.query), params },+ { query: buildBotDimensionQuery(req.query, true), params }+ );return res.send({ data: response });- } catch (error) {- console.error("Error fetching bot dimension:", error);- return res.status(500).send({ error: "Failed to fetch bot dimension" });}-}+);diff --git a/server/src/api/analytics/bots/getBotOverview.ts b/server/src/api/analytics/bots/getBotOverview.tsindex ca59204d9..fe8fda674 100644--- a/server/src/api/analytics/bots/getBotOverview.ts+++ b/server/src/api/analytics/bots/getBotOverview.ts@@ -1,7 +1,7 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement } from "./utils.js";type BotOverviewResponse = {@@ -24,10 +24,10 @@ export interface BotOverviewRequest {}>;}-const getQuery = (params: BotOverviewRequest["Querystring"]) => {- const timeStatement = getTimeStatement(params);- const filterStatement = getBotFilterStatement(params.filters);- const layerStatement = getBotLayerStatement(params.layer);+export const buildBotOverviewQuery = (query: BotOverviewRequest["Querystring"]) => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = getBotFilterStatement(query.filters);+ const layerStatement = getBotLayerStatement(query.layer);return `WITH@@ -78,20 +78,14 @@ const getQuery = (params: BotOverviewRequest["Querystring"]) => {`;};-export async function getBotOverview(req: FastifyRequest<BotOverviewRequest>, res: FastifyReply) {- try {- const result = await clickhouse.query({- query: getQuery(req.query),- format: "JSONEachRow",- query_params: {- siteId: Number(req.params.siteId),- },+export const getBotOverview = analyticsRoute<BotOverviewRequest>(+ "bot overview",+ async (req: FastifyRequest<BotOverviewRequest>, res: FastifyReply) => {+ const data = await runAnalyticsQuery<BotOverviewResponse>({+ query: buildBotOverviewQuery(req.query),+ params: { siteId: Number(req.params.siteId) },});- const data = await processResults<BotOverviewResponse>(result);return res.send({ data: data[0] });- } catch (error) {- console.error("Error fetching bot overview:", error);- return res.status(500).send({ error: "Failed to fetch bot overview" });}-}+);diff --git a/server/src/api/analytics/bots/getBotTimeSeries.ts b/server/src/api/analytics/bots/getBotTimeSeries.tsindex 2376fba6a..33df5bf84 100644--- a/server/src/api/analytics/bots/getBotTimeSeries.ts+++ b/server/src/api/analytics/bots/getBotTimeSeries.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { TimeBucket } from "../types.js";-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { type BotLayerKey, getBotFilterStatement, getBotLayerStatement, getBotTimeStatementFill } from "./utils.js";type BotTimeSeriesPoint = {@@ -21,16 +21,16 @@ export interface BotTimeSeriesRequest {}>;}-const getQuery = (params: BotTimeSeriesRequest["Querystring"]) => {- const { bucket = "hour", time_zone } = params;- const timeStatement = getTimeStatement(params);- const filterStatement = getBotFilterStatement(params.filters);- const layerStatement = getBotLayerStatement(params.layer);+export const buildBotTimeSeriesQuery = (query: BotTimeSeriesRequest["Querystring"]) => {+ const { bucket = "hour", time_zone } = query;+ const timeStatement = getTimeStatement(query);+ const filterStatement = getBotFilterStatement(query.filters);+ const layerStatement = getBotLayerStatement(query.layer);const hasBoundedTime =- Boolean(params.start_date && params.end_date) ||- Boolean(params.start_datetime && params.end_datetime) ||- (params.past_minutes_start !== undefined && params.past_minutes_end !== undefined);- const fillClause = hasBoundedTime ? getBotTimeStatementFill(params, bucket) : "";+ Boolean(query.start_date && query.end_date) ||+ Boolean(query.start_datetime && query.end_datetime) ||+ (query.past_minutes_start !== undefined && query.past_minutes_end !== undefined);+ const fillClause = hasBoundedTime ? getBotTimeStatementFill(query, bucket) : "";const timezone = SqlString.escape(time_zone || "UTC");return `@@ -47,20 +47,14 @@ const getQuery = (params: BotTimeSeriesRequest["Querystring"]) => {`;};-export async function getBotTimeSeries(req: FastifyRequest<BotTimeSeriesRequest>, res: FastifyReply) {- try {- const result = await clickhouse.query({- query: getQuery(req.query),- format: "JSONEachRow",- query_params: {- siteId: Number(req.params.siteId),- },+export const getBotTimeSeries = analyticsRoute<BotTimeSeriesRequest>(+ "bot time series",+ async (req: FastifyRequest<BotTimeSeriesRequest>, res: FastifyReply) => {+ const data = await runAnalyticsQuery<BotTimeSeriesPoint>({+ query: buildBotTimeSeriesQuery(req.query),+ params: { siteId: Number(req.params.siteId) },});- const data = await processResults<BotTimeSeriesPoint>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching bot time series:", error);- return res.status(500).send({ error: "Failed to fetch bot time series" });}-}+);diff --git a/server/src/api/analytics/events/getAutocaptureEvents.ts b/server/src/api/analytics/events/getAutocaptureEvents.tsindex ee0effad8..82c512605 100644--- a/server/src/api/analytics/events/getAutocaptureEvents.ts+++ b/server/src/api/analytics/events/getAutocaptureEvents.ts@@ -1,10 +1,10 @@import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";import { AutocaptureTargetType, isAutocaptureTargetType } from "../utils/eventConditions.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetAutocaptureEventsResponse = {value: string;@@ -33,21 +33,18 @@ const VALUE_EXPRESSIONS: Record<AutocaptureTargetType, string> = {"coalesce(nullIf(JSONExtractString(toString(props), 'formName'), ''), nullIf(JSONExtractString(toString(props), 'formId'), ''), nullIf(JSONExtractString(toString(props), 'formAction'), ''), '')",};-// Returns autocapture events of a type (button clicks, form submissions,-// copies) grouped by their display value, with counts and last occurrence.-export async function getAutocaptureEvents(req: FastifyRequest<GetAutocaptureEventsRequest>, res: FastifyReply) {- const { type, filters } = req.query;- const site = req.params.siteId;+export const buildAutocaptureEventsQuery = (+ query: GetAutocaptureEventsRequest["Querystring"],+ siteId: number,+ type: AutocaptureTargetType+) => {+ const { filters } = query;- if (!type || !isAutocaptureTargetType(type)) {- return res.status(400).send({ error: "Invalid autocapture event type" });- }-- const timeStatement = getTimeStatement(req.query);- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";+ const timeStatement = getTimeStatement(query);+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";const valueExpression = VALUE_EXPRESSIONS[type];- const query = `+ return `SELECT${valueExpression} AS value,COUNT(*) AS count,@@ -63,23 +60,27 @@ export async function getAutocaptureEvents(req: FastifyRequest<GetAutocaptureEveORDER BY count DESCLIMIT 1000`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+// Returns autocapture events of a type (button clicks, form submissions,+// copies) grouped by their display value, with counts and last occurrence.+export const getAutocaptureEvents = analyticsRoute<GetAutocaptureEventsRequest>(+ "autocapture events",+ async (req: FastifyRequest<GetAutocaptureEventsRequest>, res: FastifyReply) => {+ const { type } = req.query;+ const site = req.params.siteId;++ if (!type || !isAutocaptureTargetType(type)) {+ return res.status(400).send({ error: "Invalid autocapture event type" });+ }++ const data = await runAnalyticsQuery<GetAutocaptureEventsResponse[number]>({+ query: buildAutocaptureEventsQuery(req.query, Number(site), type),+ params: { siteId: Number(site) },});- const data = await processResults<GetAutocaptureEventsResponse[number]>(result);// processResults coerces any column that looks numeric into a number; `value`// is free-form captured text (e.g. a button labeled "100") and must stay a string.return res.send({ data: data.map(row => ({ ...row, value: String(row.value) })) });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching autocapture events:", error);- return res.status(500).send({ error: "Failed to fetch autocapture events" });}-}+);diff --git a/server/src/api/analytics/events/getAutocaptureValues.ts b/server/src/api/analytics/events/getAutocaptureValues.tsindex 03d515c71..46397cfff 100644--- a/server/src/api/analytics/events/getAutocaptureValues.ts+++ b/server/src/api/analytics/events/getAutocaptureValues.ts@@ -1,10 +1,10 @@import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { getFilterStatement } from "../utils/getFilterStatement.js";import { FilterParams } from "@rybbit/shared";-import { AUTOCAPTURE_PATTERN_PROPS, isAutocaptureTargetType } from "../utils/eventConditions.js";+import { AUTOCAPTURE_PATTERN_PROPS, AutocaptureTargetType, isAutocaptureTargetType } from "../utils/eventConditions.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetAutocaptureValuesResponse = {value: string;@@ -20,25 +20,21 @@ export interface GetAutocaptureValuesRequest {}>;}-// Returns the most common values of an autocapture type's primary props-// (outbound urls, button texts, form names/ids, copied texts), used as-// suggestions when configuring goals and funnel steps.-export async function getAutocaptureValues(req: FastifyRequest<GetAutocaptureValuesRequest>, res: FastifyReply) {- const { type, filters } = req.query;- const site = req.params.siteId;+export const buildAutocaptureValuesQuery = (+ query: GetAutocaptureValuesRequest["Querystring"],+ siteId: number,+ type: AutocaptureTargetType+) => {+ const { filters } = query;- if (!type || !isAutocaptureTargetType(type)) {- return res.status(400).send({ error: "Invalid autocapture event type" });- }-- const timeStatement = getTimeStatement(req.query);- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";+ const timeStatement = getTimeStatement(query);+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";const propExtracts = AUTOCAPTURE_PATTERN_PROPS[type].map(prop => `JSONExtractString(toString(props), ${SqlString.escape(prop)})`).join(", ");- const query = `+ return `SELECT value, COUNT(*) AS countFROM (SELECT arrayJoin([${propExtracts}]) AS value@@ -54,22 +50,28 @@ export async function getAutocaptureValues(req: FastifyRequest<GetAutocaptureValORDER BY count DESCLIMIT 500`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+// Returns the most common values of an autocapture type's primary props+// (outbound urls, button texts, form names/ids, copied texts), used as+// suggestions when configuring goals and funnel steps.+export const getAutocaptureValues = analyticsRoute<GetAutocaptureValuesRequest>(+ "autocapture values",+ async (req: FastifyRequest<GetAutocaptureValuesRequest>, res: FastifyReply) => {+ const { type } = req.query;+ const site = req.params.siteId;++ if (!type || !isAutocaptureTargetType(type)) {+ return res.status(400).send({ error: "Invalid autocapture event type" });+ }++ const data = await runAnalyticsQuery<GetAutocaptureValuesResponse[number]>({+ query: buildAutocaptureValuesQuery(req.query, Number(site), type),+ params: { siteId: Number(site) },});- const data = await processResults<GetAutocaptureValuesResponse[number]>(result);// processResults coerces any column that looks numeric into a number; `value`// is free-form captured text (e.g. a button labeled "100") and must stay a string.return res.send({ data: data.map(row => ({ ...row, value: String(row.value) })) });- } catch (error) {- console.error("Error fetching autocapture values:", error);- return res.status(500).send({ error: "Failed to fetch autocapture values" });}-}+);diff --git a/server/src/api/analytics/events/getEventBucketed.ts b/server/src/api/analytics/events/getEventBucketed.tsindex 7f956cd24..2f3980d13 100644--- a/server/src/api/analytics/events/getEventBucketed.ts+++ b/server/src/api/analytics/events/getEventBucketed.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { TimeBucket } from "../types.js";import { getFilterStatement } from "../utils/getFilterStatement.js";-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetEventBucketedResponse = {time: string;@@ -29,21 +29,13 @@ const parseLimit = (limit?: string) => {return Math.min(Math.max(Math.floor(parsed), 1), 10);};-export async function getEventBucketed(req: FastifyRequest<GetEventBucketedRequest>, res: FastifyReply) {- const site = req.params.siteId;- const { bucket = "hour", limit } = req.query;- const timeZone = req.query.time_zone || "UTC";+export const buildEventBucketedQuery = (query: GetEventBucketedRequest["Querystring"], siteId: number) => {+ const { bucket = "hour" } = query;- if (!TimeBucketToFn[bucket]) {- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });- }-- const topLimit = parseLimit(limit);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(req.query.filters, Number(site), timeStatement);-- const query = `+ return `WITH top_events AS (SELECTevent_name@@ -75,23 +67,28 @@ export async function getEventBucketed(req: FastifyRequest<GetEventBucketedRequeGROUP BY time, event_nameORDER BY time`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+export const getEventBucketed = analyticsRoute<GetEventBucketedRequest>(+ "bucketed event data",+ async (req: FastifyRequest<GetEventBucketedRequest>, res: FastifyReply) => {+ const site = req.params.siteId;+ const { bucket = "hour", limit } = req.query;+ const timeZone = req.query.time_zone || "UTC";++ if (!TimeBucketToFn[bucket]) {+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });+ }++ const data = await runAnalyticsQuery<GetEventBucketedResponse[number]>({+ query: buildEventBucketedQuery(req.query, Number(site)),+ params: {siteId: Number(site),- limit: topLimit,+ limit: parseLimit(limit),timeZone,},});- const data = await processResults<GetEventBucketedResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching bucketed event data:", error);- return res.status(500).send({ error: "Failed to fetch bucketed event data" });}-}+);diff --git a/server/src/api/analytics/events/getEventNames.ts b/server/src/api/analytics/events/getEventNames.tsindex 829f80486..f9e9a90ca 100644--- a/server/src/api/analytics/events/getEventNames.ts+++ b/server/src/api/analytics/events/getEventNames.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetEventNamesResponse = {eventName: string;@@ -18,15 +18,14 @@ export interface GetEventNamesRequest {}>;}-export async function getEventNames(req: FastifyRequest<GetEventNamesRequest>, res: FastifyReply) {- const { filters } = req.query;- const site = req.params.siteId;+export const buildEventNamesQuery = (query: GetEventNamesRequest["Querystring"], siteId: number) => {+ const { filters } = query;- const timeStatement = getTimeStatement(req.query);+ const timeStatement = getTimeStatement(query);- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";- const query = `+ return `SELECTevent_name AS eventName,count() AS count@@ -42,21 +41,18 @@ export async function getEventNames(req: FastifyRequest<GetEventNamesRequest>, rORDER BY count DESCLIMIT 1000`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+export const getEventNames = analyticsRoute<GetEventNamesRequest>(+ "event names",+ async (req: FastifyRequest<GetEventNamesRequest>, res: FastifyReply) => {+ const site = req.params.siteId;++ const data = await runAnalyticsQuery<GetEventNamesResponse[number]>({+ query: buildEventNamesQuery(req.query, Number(site)),+ params: { siteId: Number(site) },});- const data = await processResults<GetEventNamesResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching event names:", error);- return res.status(500).send({ error: "Failed to fetch event names" });}-}+);diff --git a/server/src/api/analytics/events/getEventProperties.ts b/server/src/api/analytics/events/getEventProperties.tsindex d0a93e42d..216e6ee20 100644--- a/server/src/api/analytics/events/getEventProperties.ts+++ b/server/src/api/analytics/events/getEventProperties.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetEventPropertiesResponse = {propertyKey: string;@@ -19,19 +19,14 @@ export interface GetEventPropertiesRequest {}>;}-export async function getEventProperties(req: FastifyRequest<GetEventPropertiesRequest>, res: FastifyReply) {- const { event_name: eventName, filters } = req.query;- const site = req.params.siteId;+export const buildEventPropertiesQuery = (query: GetEventPropertiesRequest["Querystring"], siteId: number) => {+ const { filters } = query;- if (!eventName) {- return res.status(400).send({ error: "Event name is required" });- }-- const timeStatement = getTimeStatement(req.query);+ const timeStatement = getTimeStatement(query);- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";- const query = `+ return `SELECTkv.1 AS propertyKey, -- Access tuple elementsreplaceRegexpAll(kv.2, '^"|"$', '') AS propertyValue, -- Remove surrounding quotes if they exist@@ -49,22 +44,26 @@ export async function getEventProperties(req: FastifyRequest<GetEventPropertiesRORDER BY propertyKey ASC, count DESCLIMIT 500`;+};++export const getEventProperties = analyticsRoute<GetEventPropertiesRequest>(+ "event properties",+ async (req: FastifyRequest<GetEventPropertiesRequest>, res: FastifyReply) => {+ const { event_name: eventName } = req.query;+ const site = req.params.siteId;- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ if (!eventName) {+ return res.status(400).send({ error: "Event name is required" });+ }++ const data = await runAnalyticsQuery<GetEventPropertiesResponse[number]>({+ query: buildEventPropertiesQuery(req.query, Number(site)),+ params: {siteId: Number(site),eventName,},});- const data = await processResults<GetEventPropertiesResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching event properties:", error);- return res.status(500).send({ error: "Failed to fetch event properties" });}-}+);diff --git a/server/src/api/analytics/events/getEvents.ts b/server/src/api/analytics/events/getEvents.tsindex 4f057b908..6777c0731 100644--- a/server/src/api/analytics/events/getEvents.ts+++ b/server/src/api/analytics/events/getEvents.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { enrichWithTraits, processResults, getTimeStatement } from "../utils/utils.js";+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, QuerySpec, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetEventsResponse = {timestamp: string;@@ -73,27 +73,16 @@ const EVENT_COLUMNS = `const EVENT_TYPE_FILTER = `AND type IN ('custom_event', 'pageview', 'outbound', 'button_click', 'copy', 'form_submit', 'input_change')`;-export async function getEvents(- req: FastifyRequest<GetEventsRequest>,- res: FastifyReply-) {- const { siteId } = req.params;- const {- since_timestamp,- before_timestamp,- page_size: pageSize = "50",- filters,- } = req.query;+export const buildEventsQuery = (query: GetEventsRequest["Querystring"], siteId: number): QuerySpec => {+ const { since_timestamp, before_timestamp, page_size: pageSize = "50", filters } = query;const limit = parseInt(pageSize, 10);- const filterStatement = filters- ? getFilterStatement(filters, Number(siteId))- : "";+ const filterStatement = filters ? getFilterStatement(filters, siteId) : "";- try {- // Mode A: Poll for new events since a timestamp (Realtime polling)- if (since_timestamp) {- const query = `+ // Mode A: Poll for new events since a timestamp (Realtime polling)+ if (since_timestamp) {+ return {+ query: `SELECT ${EVENT_COLUMNS}FROM eventsWHERE@@ -103,40 +92,30 @@ export async function getEvents(${filterStatement}ORDER BY timestamp DESCLIMIT 500- `;-- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(siteId),- sinceTimestamp: since_timestamp,- },- });-- const events = await processResults<GetEventsResponse[number]>(result);- const eventsWithTraits = await enrichWithTraits(events, Number(siteId));- return res.send({ data: eventsWithTraits });- }+ `,+ params: {+ siteId,+ sinceTimestamp: since_timestamp,+ },+ };+ }- // Mode B: Cursor-based pagination (initial load or scrolling back)- const timeStatement =- req.query.start_date || req.query.end_date- ? getTimeStatement(req.query)- : "";+ // Mode B: Cursor-based pagination (initial load or scrolling back)+ const timeStatement = query.start_date || query.end_date ? getTimeStatement(query) : "";- let cursorCondition = "";- const queryParams: Record<string, string | number> = {- siteId: Number(siteId),- limit: Number(limit),- };+ let cursorCondition = "";+ const queryParams: Record<string, string | number> = {+ siteId,+ limit: Number(limit),+ };- if (before_timestamp) {- cursorCondition = `AND timestamp < toDateTime64({beforeTimestamp:String}, 3)`;- queryParams.beforeTimestamp = before_timestamp;- }+ if (before_timestamp) {+ cursorCondition = `AND timestamp < toDateTime64({beforeTimestamp:String}, 3)`;+ queryParams.beforeTimestamp = before_timestamp;+ }- const query = `+ return {+ query: `SELECT ${EVENT_COLUMNS}FROM eventsWHERE@@ -147,27 +126,33 @@ export async function getEvents(${filterStatement}ORDER BY timestamp DESCLIMIT {limit:Int32}- `;+ `,+ params: queryParams,+ };+};- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: queryParams,- });+export const getEvents = analyticsRoute<GetEventsRequest>(+ "events",+ async (req: FastifyRequest<GetEventsRequest>, res: FastifyReply) => {+ const { siteId } = req.params;+ const { since_timestamp, page_size: pageSize = "50" } = req.query;- const events = await processResults<GetEventsResponse[number]>(result);+ const events = await runAnalyticsQuery<GetEventsResponse[number]>(buildEventsQuery(req.query, Number(siteId)));const eventsWithTraits = await enrichWithTraits(events, Number(siteId));+ // Mode A: Poll for new events since a timestamp (Realtime polling)+ if (since_timestamp) {+ return res.send({ data: eventsWithTraits });+ }++ // Mode B: Cursor-based pagination (initial load or scrolling back)+ const limit = parseInt(pageSize, 10);return res.send({data: eventsWithTraits,cursor: {hasMore: events.length === limit,- oldestTimestamp:- events.length > 0 ? events[events.length - 1].timestamp : null,+ oldestTimestamp: events.length > 0 ? events[events.length - 1].timestamp : null,},});- } catch (error) {- console.error("Error fetching events:", error);- return res.status(500).send({ error: "Failed to fetch events" });}-}+);diff --git a/server/src/api/analytics/events/getOutboundLinks.ts b/server/src/api/analytics/events/getOutboundLinks.tsindex 2bc4398c9..2c83a1344 100644--- a/server/src/api/analytics/events/getOutboundLinks.ts+++ b/server/src/api/analytics/events/getOutboundLinks.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetOutboundLinksResponse = {url: string;@@ -17,14 +17,13 @@ export interface GetOutboundLinksRequest {Querystring: FilterParams<{}>;}-export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksRequest>, res: FastifyReply) {- const { filters } = req.query;- const site = req.params.siteId;+export const buildOutboundLinksQuery = (query: GetOutboundLinksRequest["Querystring"], siteId: number) => {+ const { filters } = query;- const timeStatement = getTimeStatement(req.query);- const filterStatement = filters ? getFilterStatement(filters, Number(site), timeStatement) : "";+ const timeStatement = getTimeStatement(query);+ const filterStatement = filters ? getFilterStatement(filters, siteId, timeStatement) : "";- const query = `+ return `SELECTJSONExtractString(toString(props), 'url') AS url,COUNT(*) AS count,@@ -39,21 +38,18 @@ export async function getOutboundLinks(req: FastifyRequest<GetOutboundLinksRequeORDER BY count DESCLIMIT 1000`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+export const getOutboundLinks = analyticsRoute<GetOutboundLinksRequest>(+ "outbound links",+ async (req: FastifyRequest<GetOutboundLinksRequest>, res: FastifyReply) => {+ const site = req.params.siteId;++ const data = await runAnalyticsQuery<GetOutboundLinksResponse[number]>({+ query: buildOutboundLinksQuery(req.query, Number(site)),+ params: { siteId: Number(site) },});- const data = await processResults<GetOutboundLinksResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching outbound links:", error);- return res.status(500).send({ error: "Failed to fetch outbound links" });}-}+);diff --git a/server/src/api/analytics/events/getSiteEventCount.ts b/server/src/api/analytics/events/getSiteEventCount.tsindex f661b116b..affdd2c6b 100644--- a/server/src/api/analytics/events/getSiteEventCount.ts+++ b/server/src/api/analytics/events/getSiteEventCount.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { TimeBucket } from "../types.js";import { getFilterStatement } from "../utils/getFilterStatement.js";-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetSiteEventCountResponse = {time: string;@@ -28,22 +28,13 @@ interface GetSiteEventCountRequest {}>;}-export async function getSiteEventCount(- req: FastifyRequest<GetSiteEventCountRequest>,- res: FastifyReply-) {- const site = req.params.siteId;- const { bucket = "day" } = req.query;- const timeZone = req.query.time_zone || "UTC";+export const buildSiteEventCountQuery = (query: GetSiteEventCountRequest["Querystring"], siteId: number) => {+ const { bucket = "day" } = query;- if (!TimeBucketToFn[bucket]) {- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });- }-- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(req.query.filters, Number(site), timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);- const query = `+ return `SELECTtoDateTime(${TimeBucketToFn[bucket]}(toTimeZone(timestamp, {timeZone:String}))) AS time,countIf(type = 'pageview') as pageview_count,@@ -65,22 +56,27 @@ export async function getSiteEventCount(GROUP BY timeORDER BY time`;+};++export const getSiteEventCount = analyticsRoute<GetSiteEventCountRequest>(+ "site event count",+ async (req: FastifyRequest<GetSiteEventCountRequest>, res: FastifyReply) => {+ const site = req.params.siteId;+ const { bucket = "day" } = req.query;+ const timeZone = req.query.time_zone || "UTC";- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ if (!TimeBucketToFn[bucket]) {+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });+ }++ const data = await runAnalyticsQuery<GetSiteEventCountResponse[number]>({+ query: buildSiteEventCountQuery(req.query, Number(site)),+ params: {siteId: Number(site),timeZone,},});- const data = await processResults<GetSiteEventCountResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching site event count:", error);- return res.status(500).send({ error: "Failed to fetch site event count" });}-}+);diff --git a/server/src/api/analytics/funnels/getFunnel.ts b/server/src/api/analytics/funnels/getFunnel.tsindex bf8f042d9..976de1614 100644--- a/server/src/api/analytics/funnels/getFunnel.ts+++ b/server/src/api/analytics/funnels/getFunnel.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { AnalyticsQueryError, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { buildFunnelStepCondition, FunnelStep } from "./funnelSteps.js";type Funnel = {@@ -18,33 +18,15 @@ type FunnelResponse = {dropoff_rate: number;};-export async function getFunnel(- request: FastifyRequest<{- Body: Funnel;- Params: {- siteId: string;- };- Querystring: FilterParams<{}>;- }>,- reply: FastifyReply-) {- const { steps } = request.body;- const { siteId } = request.params;-- // Validate request- if (!steps || steps.length < 2) {- return reply.status(400).send({ error: "At least 2 steps are required for a funnel" });- }+export const buildFunnelQuery = (query: FilterParams<{}>, siteId: number, steps: FunnelStep[]) => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);- try {- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(request.query.filters, Number(siteId), timeStatement);+ // Build conditional statements for each step+ const stepConditions = steps.map(step => buildFunnelStepCondition(step));- // Build conditional statements for each step- const stepConditions = steps.map(step => buildFunnelStepCondition(step));-- // Build the funnel query - session-based tracking- const query = `+ // Build the funnel query - session-based tracking+ return `WITH-- Get all session actions in the time periodSessionActions AS (@@ -107,14 +89,14 @@ export async function getFunnel().join("\nUNION ALL\n")})-+-- Final results with calculated conversion and dropoff ratesSELECTs1.step_number,s1.step_name,s1.visitors as visitors,round(s1.visitors * 100.0 / first_step.visitors, 2) as conversion_rate,- CASE+ CASEWHEN s1.step_number = 1 THEN 0ELSE round((1 - (s1.visitors / prev_step.visitors)) * 100.0, 2)END as dropoff_rate@@ -127,22 +109,45 @@ export async function getFunnel() as prev_step ON s1.step_number = prev_step.next_step_numberORDER BY s1.step_number`;+};++export async function getFunnel(+ request: FastifyRequest<{+ Body: Funnel;+ Params: {+ siteId: string;+ };+ Querystring: FilterParams<{}>;+ }>,+ reply: FastifyReply+) {+ const { steps } = request.body;+ const { siteId } = request.params;++ // Validate request+ if (!steps || steps.length < 2) {+ return reply.status(400).send({ error: "At least 2 steps are required for a funnel" });+ }- // Execute the query- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ try {+ const data = await runAnalyticsQuery<FunnelResponse>({+ query: buildFunnelQuery(request.query, Number(siteId), steps),+ params: {siteId: Number(siteId),stepNumber: steps.length,},});- // Process the results- const data = await processResults<FunnelResponse>(result);return reply.send({ data });} catch (error) {- console.error("Error executing funnel query:", error);+ if (error instanceof AnalyticsQueryError) {+ console.error("Error executing funnel query:", error.original);+ for (const query of error.queries) {+ console.error("Failed query:", query);+ }+ } else {+ console.error("Error executing funnel query:", error);+ }return reply.status(500).send({ error: "Failed to execute funnel analysis" });}}diff --git a/server/src/api/analytics/funnels/getFunnelStepSessions.ts b/server/src/api/analytics/funnels/getFunnelStepSessions.tsindex c51800f78..6272f5387 100644--- a/server/src/api/analytics/funnels/getFunnelStepSessions.ts+++ b/server/src/api/analytics/funnels/getFunnelStepSessions.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";import { GetSessionsResponse } from "../sessions/getSessions.js";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { buildFunnelStepCondition, FunnelStep } from "./funnelSteps.js";type Funnel = {@@ -23,47 +23,28 @@ export interface GetFunnelStepSessionsRequest {}>;}-export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSessionsRequest>, res: FastifyReply) {- const { steps } = req.body;- const { stepNumber: stepNumberStr, siteId } = req.params;- const { mode, page, limit } = req.query;-- const stepNumber = parseInt(stepNumberStr, 10);-- // Validate request- if (!steps || steps.length < 2) {- return res.status(400).send({ error: "At least 2 steps are required for a funnel" });- }-- if (isNaN(stepNumber) || stepNumber < 1 || stepNumber > steps.length) {- return res.status(400).send({ error: "Invalid step number" });- }-- if (mode !== "reached" && mode !== "dropped") {- return res.status(400).send({ error: "Mode must be 'reached' or 'dropped'" });- }-- // For final step in "dropped" mode, return empty results (no drop-off possible)- if (mode === "dropped" && stepNumber === steps.length) {- return res.send({ data: [] });- }-- try {- const timeStatement = getTimeStatement(req.query);-- // Applied once, in the SessionActions CTE against raw events (same as the- // funnel analyze endpoint), where every filter parameter's column exists.- // Re-applying it to the aggregated outer query breaks on parameters the- // aggregate doesn't project (utm_*, pathname, timezone → unknown identifier).- const filterStatement = getFilterStatement(req.query.filters, Number(siteId), timeStatement);-- // Build conditional statements for each step we need- const stepsToCheck = mode === "reached" ? stepNumber : stepNumber + 1;- const stepConditions = steps.slice(0, stepsToCheck).map(step => buildFunnelStepCondition(step));-- // Build CTEs for each funnel step to identify qualifying sessions- const stepCTEs = [- `+export const buildFunnelStepSessionsQuery = (+ query: GetFunnelStepSessionsRequest["Querystring"],+ siteId: number,+ steps: FunnelStep[],+ stepNumber: number+) => {+ const { mode } = query;+ const timeStatement = getTimeStatement(query);++ // Applied once, in the SessionActions CTE against raw events (same as the+ // funnel analyze endpoint), where every filter parameter's column exists.+ // Re-applying it to the aggregated outer query breaks on parameters the+ // aggregate doesn't project (utm_*, pathname, timezone → unknown identifier).+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);++ // Build conditional statements for each step we need+ const stepsToCheck = mode === "reached" ? stepNumber : stepNumber + 1;+ const stepConditions = steps.slice(0, stepsToCheck).map(step => buildFunnelStepCondition(step));++ // Build CTEs for each funnel step to identify qualifying sessions+ const stepCTEs = [+ `SessionActions AS (SELECTsession_id,@@ -89,11 +70,11 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSesWHERE ${stepConditions[0]}GROUP BY session_id)`,- ];+ ];- // Add CTEs for steps 2 through stepNumber (and +1 for dropped mode)- for (let i = 1; i < stepsToCheck; i++) {- stepCTEs.push(`+ // Add CTEs for steps 2 through stepNumber (and +1 for dropped mode)+ for (let i = 1; i < stepsToCheck; i++) {+ stepCTEs.push(`Step${i + 1} AS (SELECT DISTINCTs${i}.session_id,@@ -105,20 +86,20 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSesAND ${stepConditions[i]}GROUP BY s${i}.session_id)`);- }+ }- // Determine which sessions to retrieve- let targetSessionsCTE = "";- if (mode === "reached") {- // Sessions that completed step N- targetSessionsCTE = `+ // Determine which sessions to retrieve+ let targetSessionsCTE = "";+ if (mode === "reached") {+ // Sessions that completed step N+ targetSessionsCTE = `TargetSessions AS (SELECT session_idFROM Step${stepNumber})`;- } else {- // Sessions that completed step N but NOT step N+1- targetSessionsCTE = `+ } else {+ // Sessions that completed step N but NOT step N+1+ targetSessionsCTE = `TargetSessions AS (SELECT session_idFROM Step${stepNumber}@@ -127,10 +108,10 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSesFROM Step${stepNumber + 1}))`;- }+ }- // Build main query to aggregate session data- const query = `+ // Build main query to aggregate session data+ return `WITH${stepCTEs.join(",\n")},@@ -189,22 +170,45 @@ export async function getFunnelStepSessions(req: FastifyRequest<GetFunnelStepSesFROM AggregatedSessionsLIMIT {limit:Int32} OFFSET {offset:Int32}`;+};++export const getFunnelStepSessions = analyticsRoute<GetFunnelStepSessionsRequest>(+ "funnel step sessions",+ async (req: FastifyRequest<GetFunnelStepSessionsRequest>, res: FastifyReply) => {+ const { steps } = req.body;+ const { stepNumber: stepNumberStr, siteId } = req.params;+ const { mode, page, limit } = req.query;++ const stepNumber = parseInt(stepNumberStr, 10);++ // Validate request+ if (!steps || steps.length < 2) {+ return res.status(400).send({ error: "At least 2 steps are required for a funnel" });+ }- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ if (isNaN(stepNumber) || stepNumber < 1 || stepNumber > steps.length) {+ return res.status(400).send({ error: "Invalid step number" });+ }++ if (mode !== "reached" && mode !== "dropped") {+ return res.status(400).send({ error: "Mode must be 'reached' or 'dropped'" });+ }++ // For final step in "dropped" mode, return empty results (no drop-off possible)+ if (mode === "dropped" && stepNumber === steps.length) {+ return res.send({ data: [] });+ }++ const data = await runAnalyticsQuery<Omit<GetSessionsResponse[number], "traits">>({+ query: buildFunnelStepSessionsQuery(req.query, Number(siteId), steps, stepNumber),+ params: {siteId: Number(siteId),limit: limit || 25,offset: ((page || 1) - 1) * (limit || 25),},});- const data = await processResults<Omit<GetSessionsResponse[number], "traits">>(result);const dataWithTraits = await enrichWithTraits(data, Number(siteId));return res.send({ data: dataWithTraits });- } catch (error) {- console.error("Error fetching funnel step sessions:", error);- return res.status(500).send({ error: "Failed to fetch funnel step sessions" });}-}+);diff --git a/server/src/api/analytics/getErrorBucketed.ts b/server/src/api/analytics/getErrorBucketed.tsindex 4ed7029b5..aba98c669 100644--- a/server/src/api/analytics/getErrorBucketed.ts+++ b/server/src/api/analytics/getErrorBucketed.ts@@ -1,11 +1,11 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { validateTimeStatementFillParams } from "./utils/query-validation.js";-import { getTimeStatement, TimeBucketToFn, bucketIntervalMap, processResults } from "./utils/utils.js";+import { getTimeStatement, TimeBucketToFn, bucketIntervalMap } from "./utils/utils.js";import { TimeBucket } from "./types.js";import { getFilterStatement } from "./utils/getFilterStatement.js";+import { AnalyticsQueryError, runAnalyticsQuery } from "./utils/analyticsQuery.js";function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);@@ -58,21 +58,13 @@ export type GetErrorBucketedResponse = {error_count: number;}[];-export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedRequest>, res: FastifyReply) {- const site = req.params.siteId;- const { bucket, errorMessage } = req.query;-- if (!errorMessage) {- return res.status(400).send({ error: "errorMessage parameter is required" });- }-- const numericSiteId = Number(site);- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(req.query.filters, numericSiteId, timeStatement);- const timeStatementFill = getTimeStatementFill(req.query, bucket);+export const buildErrorBucketedQuery = (query: GetErrorBucketedRequest["Querystring"], siteId: number) => {+ const { bucket } = query;+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);+ const timeStatementFill = getTimeStatementFill(query, bucket);- try {- const query = `+ return `SELECT${TimeBucketToFn[bucket]}(toTimeZone(timestamp, {timeZone:String})) AS time,COUNT(*) AS error_count@@ -87,25 +79,34 @@ export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedRequeORDER BY time${timeStatementFill}`;+};++export async function getErrorBucketed(req: FastifyRequest<GetErrorBucketedRequest>, res: FastifyReply) {+ const site = req.params.siteId;+ const { errorMessage } = req.query;++ if (!errorMessage) {+ return res.status(400).send({ error: "errorMessage parameter is required" });+ }++ const numericSiteId = Number(site);- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ try {+ const data = await runAnalyticsQuery<GetErrorBucketedResponse[number]>({+ query: buildErrorBucketedQuery(req.query, numericSiteId),+ params: {siteId: numericSiteId,errorMessage: errorMessage,timeZone: req.query.time_zone || "UTC",},});- const data = await processResults<GetErrorBucketedResponse>(result);-return res.send({success: true,data: data,});} catch (error) {- console.error("Error getting error bucketed data:", error);+ console.error("Error getting error bucketed data:", error instanceof AnalyticsQueryError ? error.original : error);return res.status(500).send({success: false,error: "Failed to get error data",diff --git a/server/src/api/analytics/getErrorEvents.ts b/server/src/api/analytics/getErrorEvents.tsindex 548f3548b..7dd923c09 100644--- a/server/src/api/analytics/getErrorEvents.ts+++ b/server/src/api/analytics/getErrorEvents.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "./utils/utils.js";+import { getTimeStatement } from "./utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "./utils/getFilterStatement.js";+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";interface GetErrorEventsRequest {Params: {@@ -46,31 +46,18 @@ type ErrorEventsPaginatedResponse = {totalCount: number;};-const getErrorEventsQuery = (request: FastifyRequest<GetErrorEventsRequest>, isCountQuery: boolean = false) => {- const { filters, limit, page } = request.query;+export const buildErrorEventsQuery = (+ query: GetErrorEventsRequest["Querystring"],+ siteId: number,+ isCountQuery: boolean = false+) => {+ const { filters } = query;- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);- let validatedLimit: number | null = null;- if (!isCountQuery && limit !== undefined) {- const parsedLimit = parseInt(String(limit), 10);- if (!isNaN(parsedLimit) && parsedLimit > 0) {- validatedLimit = parsedLimit;- }- }// Default to 20 for error events- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 20";-- let validatedOffset: number | null = null;- if (!isCountQuery && page !== undefined) {- const parsedPage = parseInt(String(page), 10);- if (!isNaN(parsedPage) && parsedPage >= 1) {- const pageOffset = (parsedPage - 1) * (validatedLimit || 20);- validatedOffset = pageOffset;- }- }- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 20, isCountQuery);if (isCountQuery) {return `@@ -131,52 +118,28 @@ const getErrorEventsQuery = (request: FastifyRequest<GetErrorEventsRequest>, isC`;};-export async function getErrorEvents(req: FastifyRequest<GetErrorEventsRequest>, res: FastifyReply) {- const site = req.params.siteId;- const { errorMessage, page } = req.query;-- if (!errorMessage) {- return res.status(400).send({ error: "errorMessage parameter is required" });- }-- const isPaginatedRequest = page !== undefined;+export const getErrorEvents = analyticsRoute<GetErrorEventsRequest>(+ "error events",+ async (req: FastifyRequest<GetErrorEventsRequest>, res: FastifyReply) => {+ const { errorMessage } = req.query;- const dataQuery = getErrorEventsQuery(req, false);+ if (!errorMessage) {+ return res.status(400).send({ error: "errorMessage parameter is required" });+ }- try {- const dataResult = await clickhouse.query({- query: dataQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- errorMessage: errorMessage,- },- });- const items = await processResults<ErrorEvent>(dataResult);+ const siteId = Number(req.params.siteId);+ const params = { siteId, errorMessage };+ const dataSpec = { query: buildErrorEventsQuery(req.query, siteId, false), params };- if (isPaginatedRequest) {- const countQuery = getErrorEventsQuery(req, true);- const countResult = await clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- errorMessage: errorMessage,- },+ if (req.query.page !== undefined) {+ const result = await runPaginatedQuery<ErrorEvent>(dataSpec, {+ query: buildErrorEventsQuery(req.query, siteId, true),+ params,});- const countData = await processResults<{ totalCount: number }>(countResult);- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;- return res.send({ data: { data: items, totalCount } });- } else {- return res.send({ data: items });+ return res.send({ data: result });}- } catch (error) {- console.error(`Error fetching error events:`, error);- console.error("Failed dataQuery:", dataQuery);- if (isPaginatedRequest) {- const countQuery = getErrorEventsQuery(req, true);- console.error("Failed countQuery:", countQuery);- }- return res.status(500).send({ error: `Failed to fetch error events` });++ const items = await runAnalyticsQuery<ErrorEvent>(dataSpec);+ return res.send({ data: items });}-}+);diff --git a/server/src/api/analytics/getErrorNames.ts b/server/src/api/analytics/getErrorNames.tsindex 0426006a2..bff0056fb 100644--- a/server/src/api/analytics/getErrorNames.ts+++ b/server/src/api/analytics/getErrorNames.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "./utils/utils.js";+import { getTimeStatement } from "./utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "./utils/getFilterStatement.js";+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";interface GetErrorNamesRequest {Params: {@@ -29,31 +29,18 @@ type ErrorNamesPaginatedResponse = {totalCount: number;};-const getErrorNamesQuery = (request: FastifyRequest<GetErrorNamesRequest>, isCountQuery: boolean = false) => {- const { filters, limit, page } = request.query;+export const buildErrorNamesQuery = (+ query: GetErrorNamesRequest["Querystring"],+ siteId: number,+ isCountQuery: boolean = false+) => {+ const { filters } = query;- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);- let validatedLimit: number | null = null;- if (!isCountQuery && limit !== undefined) {- const parsedLimit = parseInt(String(limit), 10);- if (!isNaN(parsedLimit) && parsedLimit > 0) {- validatedLimit = parsedLimit;- }- }// Default to 10 for non-paginated use- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 10";-- let validatedOffset: number | null = null;- if (!isCountQuery && page !== undefined) {- const parsedPage = parseInt(String(page), 10);- if (!isNaN(parsedPage) && parsedPage >= 1) {- const pageOffset = (parsedPage - 1) * (validatedLimit || 10);- validatedOffset = pageOffset;- }- }- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 10, isCountQuery);// For errors, we want to count total occurrences and unique sessions affected// Group by error message instead of error name@@ -103,47 +90,23 @@ const getErrorNamesQuery = (request: FastifyRequest<GetErrorNamesRequest>, isCou`;};-export async function getErrorNames(req: FastifyRequest<GetErrorNamesRequest>, res: FastifyReply) {- const site = req.params.siteId;- const { page } = req.query;-- const isPaginatedRequest = page !== undefined; // True if page is present+export const getErrorNames = analyticsRoute<GetErrorNamesRequest>(+ "error names",+ async (req: FastifyRequest<GetErrorNamesRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const params = { siteId };+ const dataSpec = { query: buildErrorNamesQuery(req.query, siteId, false), params };- const dataQuery = getErrorNamesQuery(req, false);-- try {- const dataResult = await clickhouse.query({- query: dataQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- });- const items = await processResults<ErrorNameItem>(dataResult);-- if (isPaginatedRequest) {- const countQuery = getErrorNamesQuery(req, true);- const countResult = await clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ if (req.query.page !== undefined) {+ const result = await runPaginatedQuery<ErrorNameItem>(dataSpec, {+ query: buildErrorNamesQuery(req.query, siteId, true),+ params,});- const countData = await processResults<{ totalCount: number }>(countResult);- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;- return res.send({ data: { data: items, totalCount } });- } else {- // For non-paginated (StandardSection default) use, return the simpler structure- return res.send({ data: items });- }- } catch (error) {- console.error(`Error fetching error names:`, error);- console.error("Failed dataQuery:", dataQuery);- if (isPaginatedRequest) {- const countQuery = getErrorNamesQuery(req, true);- console.error("Failed countQuery:", countQuery);+ return res.send({ data: result });}- return res.status(500).send({ error: `Failed to fetch error names` });++ // For non-paginated (StandardSection default) use, return the simpler structure+ const items = await runAnalyticsQuery<ErrorNameItem>(dataSpec);+ return res.send({ data: items });}-}+);diff --git a/server/src/api/analytics/getJourneys.ts b/server/src/api/analytics/getJourneys.tsindex 919ee85f3..af5bff118 100644--- a/server/src/api/analytics/getJourneys.ts+++ b/server/src/api/analytics/getJourneys.ts@@ -2,9 +2,9 @@ import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";import { z } from "zod";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { getFilterStatement } from "./utils/getFilterStatement.js";import { getTimeStatement, patternToRegex } from "./utils/utils.js";+import { AnalyticsQueryError, runAnalyticsQuery } from "./utils/analyticsQuery.js";// stepFilters arrives as a JSON object mapping a (numeric) step index to a path// pattern. Both the keys and values are attacker-controlled, so validate the@@ -14,71 +14,48 @@ const stepFiltersSchema = z.record(z.string().max(2048));-export const getJourneys = async (- request: FastifyRequest<{- Params: { siteId: string };- Querystring: FilterParams<{- steps?: string;- limit?: string;- stepFilters?: string;- }>;- }>,- reply: FastifyReply-) => {- try {- const { siteId } = request.params;- const { steps = "3", limit = "100", filters, stepFilters } = request.query;-- const maxSteps = parseInt(steps, 10);- const journeyLimit = parseInt(limit, 10);-- if (isNaN(maxSteps) || maxSteps < 2 || maxSteps > 10) {- return reply.status(400).send({- error: "Steps parameter must be a number between 2 and 10",- });- }+interface GetJourneysRequest {+ Params: { siteId: string };+ Querystring: FilterParams<{+ steps?: string;+ limit?: string;+ stepFilters?: string;+ }>;+}- if (isNaN(journeyLimit) || journeyLimit < 1 || journeyLimit > 500) {- return reply.status(400).send({- error: "Limit parameter must be a number between 1 and 500",- });- }+type JourneyRow = {+ journey: string[];+ sessions_count: number;+ percentage: number;+};- // Time conditions using getTimeStatement- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(filters, Number(siteId), timeStatement);+export const buildJourneysQuery = (+ query: GetJourneysRequest["Querystring"],+ siteId: number,+ parsedStepFilters: Record<string, string>+) => {+ // Time conditions using getTimeStatement+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);- // Parse and validate step filters- let parsedStepFilters: Record<string, string> = {};- if (stepFilters) {- try {- parsedStepFilters = stepFiltersSchema.parse(JSON.parse(stepFilters));- } catch (error) {- return reply.status(400).send({- error: "Invalid stepFilters format",- });+ // Build step filter conditions for the HAVING clause+ // Supports wildcard patterns: * matches single segment, ** matches multiple segments+ const stepFilterConditions = Object.entries(parsedStepFilters)+ .map(([step, path]) => {+ const stepIndex = parseInt(step, 10) + 1; // ClickHouse arrays are 1-indexed+ if (path.includes("*")) {+ // Use regex matching for wildcard patterns. SqlString.escape correctly+ // escapes the regex literal for ClickHouse (handles both ' and \).+ const regex = patternToRegex(path);+ return `match(journey[${stepIndex}], ${SqlString.escape(regex)})`;}- }-- // Build step filter conditions for the HAVING clause- // Supports wildcard patterns: * matches single segment, ** matches multiple segments- const stepFilterConditions = Object.entries(parsedStepFilters)- .map(([step, path]) => {- const stepIndex = parseInt(step, 10) + 1; // ClickHouse arrays are 1-indexed- if (path.includes("*")) {- // Use regex matching for wildcard patterns. SqlString.escape correctly- // escapes the regex literal for ClickHouse (handles both ' and \).- const regex = patternToRegex(path);- return `match(journey[${stepIndex}], ${SqlString.escape(regex)})`;- }- // Use exact match for non-wildcard patterns (more efficient)- return `journey[${stepIndex}] = ${SqlString.escape(path)}`;- })- .join(" AND ");+ // Use exact match for non-wildcard patterns (more efficient)+ return `journey[${stepIndex}] = ${SqlString.escape(path)}`;+ })+ .join(" AND ");- // Query to find sequences of events (journeys) for each user- const result = await clickhouse.query({- query: `+ // Query to find sequences of events (journeys) for each user+ return `WITH user_paths AS (SELECTsession_id,@@ -122,25 +99,59 @@ export const getJourneys = async (${filterStatement || ""}) AS percentageFROM journey_segments- `,- query_params: {+ `;+};++export const getJourneys = async (request: FastifyRequest<GetJourneysRequest>, reply: FastifyReply) => {+ try {+ const { siteId } = request.params;+ const { steps = "3", limit = "100", stepFilters } = request.query;++ const maxSteps = parseInt(steps, 10);+ const journeyLimit = parseInt(limit, 10);++ if (isNaN(maxSteps) || maxSteps < 2 || maxSteps > 10) {+ return reply.status(400).send({+ error: "Steps parameter must be a number between 2 and 10",+ });+ }++ if (isNaN(journeyLimit) || journeyLimit < 1 || journeyLimit > 500) {+ return reply.status(400).send({+ error: "Limit parameter must be a number between 1 and 500",+ });+ }++ // Parse and validate step filters+ let parsedStepFilters: Record<string, string> = {};+ if (stepFilters) {+ try {+ parsedStepFilters = stepFiltersSchema.parse(JSON.parse(stepFilters));+ } catch (error) {+ return reply.status(400).send({+ error: "Invalid stepFilters format",+ });+ }+ }++ const data = await runAnalyticsQuery<JourneyRow>({+ query: buildJourneysQuery(request.query, Number(siteId), parsedStepFilters),+ params: {siteId: parseInt(siteId, 10),maxSteps: maxSteps,journeyLimit: journeyLimit,},});- const data = await result.json();-return reply.send({- journeys: data.data.map((item: any) => ({+ journeys: data.map(item => ({path: item.journey,count: Number(item.sessions_count),percentage: Number(item.percentage),})),});} catch (error) {- console.error("Error getting journeys:", error);+ console.error("Error getting journeys:", error instanceof AnalyticsQueryError ? error.original : error);return reply.status(500).send({ error: "Failed to get journeys" });}};diff --git a/server/src/api/analytics/getLiveUsercount.ts b/server/src/api/analytics/getLiveUsercount.tsindex 37a11048b..8da81630d 100644--- a/server/src/api/analytics/getLiveUsercount.ts+++ b/server/src/api/analytics/getLiveUsercount.ts@@ -1,27 +1,28 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { processResults } from "./utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";-export const getLiveUsercount = async (- req: FastifyRequest<{- Params: { siteId: string };- Querystring: { minutes: number };- }>,- res: FastifyReply-) => {- const { siteId } = req.params;- const { minutes } = req.query;+interface GetLiveUsercountRequest {+ Params: { siteId: string };+ Querystring: { minutes: number };+}- const query = await clickhouse.query({- query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval {minutes:Int32} minute AND site_id = {siteId:Int32}`,- format: "JSONEachRow",- query_params: {- siteId: Number(siteId),- minutes: Number(minutes || 5),- },- });+export const buildLiveUsercountQuery = () =>+ `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval {minutes:Int32} minute AND site_id = {siteId:Int32}`;- const result = await processResults<{ count: number }>(query);+export const getLiveUsercount = analyticsRoute<GetLiveUsercountRequest>(+ "live user count",+ async (req: FastifyRequest<GetLiveUsercountRequest>, res: FastifyReply) => {+ const { siteId } = req.params;+ const { minutes } = req.query;- return res.send({ count: result[0].count });-};+ const result = await runAnalyticsQuery<{ count: number }>({+ query: buildLiveUsercountQuery(),+ params: {+ siteId: Number(siteId),+ minutes: Number(minutes || 5),+ },+ });++ return res.send({ count: result[0].count });+ }+);diff --git a/server/src/api/analytics/getMetric.test.ts b/server/src/api/analytics/getMetric.test.tsnew file mode 100644index 000000000..bbef713d0--- /dev/null+++ b/server/src/api/analytics/getMetric.test.ts@@ -0,0 +1,132 @@+import { describe, expect, it, vi } from "vitest";++vi.mock("../../db/clickhouse/clickhouse.js", () => ({+ clickhouse: { query: vi.fn() },+}));+vi.mock("../../db/postgres/postgres.js", () => ({+ db: {},+}));++import { buildMetricQuery } from "./getMetric.js";+import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";+import { FilterParameter } from "./types.js";++const baseQuery = (overrides: Partial<Record<string, unknown>> = {}) =>+ ({+ start_date: "",+ end_date: "",+ time_zone: "",+ filters: "",+ parameter: "browser" as FilterParameter,+ ...overrides,+ }) as Parameters<typeof buildMetricQuery>[0];++const SITE_ID = 1;++describe("buildMetricQuery", () => {+ describe("default (session-attributed) parameters", () => {+ it("aggregates the parameter per session with argMin", () => {+ const sql = buildMetricQuery(baseQuery(), SITE_ID);+ expect(sql).toContain("argMin(browser, e.timestamp) as value");+ expect(sql).toContain("GROUP BY e.session_id");+ expect(sql).toContain("LIMIT 100");+ });++ it("uses the session-channel aggregation for channel", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "channel" }), SITE_ID);+ expect(sql).toContain(`${SESSION_CHANNEL_AGG} as value`);+ expect(sql).not.toContain("argMin(channel, e.timestamp)");+ });++ it("counts distinct values in the count query without pagination", () => {+ const sql = buildMetricQuery(baseQuery({ limit: 25, page: 3 }), SITE_ID, true);+ expect(sql).toContain("COUNT(DISTINCT value) as totalCount");+ expect(sql).not.toContain("LIMIT");+ expect(sql).not.toContain("OFFSET");+ });+ });++ describe("event_name", () => {+ it("restricts to custom events", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "event_name" }), SITE_ID);+ expect(sql).toContain("event_name as value");+ expect(sql).toContain("AND type = 'custom_event'");+ });++ it("counts distinct event names", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "event_name" }), SITE_ID, true);+ expect(sql).toContain("COUNT(DISTINCT event_name) as totalCount");+ });+ });++ describe("page_title", () => {+ it("computes bounce rate from single-pageview sessions", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "page_title" }), SITE_ID);+ expect(sql).toContain("SessionPageCounts");+ expect(sql).toContain("pageviews_in_session = 1");+ expect(sql).toContain("as bounce_rate");+ });++ it("wraps the core logic for the count query", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "page_title" }), SITE_ID, true);+ expect(sql).toContain("SELECT COUNT(*) as totalCount FROM (");+ });+ });++ describe("entry_page / exit_page", () => {+ it("orders sessions ascending for entry pages", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);+ expect(sql).toContain("ORDER BY timestamp ASC) as row_num");+ expect(sql).toContain("WHERE row_num = 1");+ });++ it("orders sessions descending for exit pages", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "exit_page" }), SITE_ID);+ expect(sql).toContain("ORDER BY timestamp DESC) as row_num");+ });++ it("caps time-on-page at 30 minutes", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "entry_page" }), SITE_ID);+ expect(sql).toContain("time_diff_seconds > 1800, 1800");+ });+ });++ describe("pathname", () => {+ it("aggregates per-path stats with hostname", () => {+ const sql = buildMetricQuery(baseQuery({ parameter: "pathname" }), SITE_ID);+ expect(sql).toContain("anyHeavy(hostname) as top_hostname");+ expect(sql).toContain("GROUP BY pathname");+ });+ });++ describe("pagination", () => {+ it("applies explicit limit and page offsets", () => {+ const sql = buildMetricQuery(baseQuery({ limit: 7, page: 3 }), SITE_ID);+ expect(sql).toContain("LIMIT 7");+ expect(sql).toContain("OFFSET 14");+ });++ it("omits OFFSET on the first page", () => {+ const sql = buildMetricQuery(baseQuery({ limit: 7, page: 1 }), SITE_ID);+ expect(sql).toContain("LIMIT 7");+ expect(sql).not.toContain("OFFSET");+ });+ });++ describe("time and filter statements", () => {+ it("constrains by the requested date range", () => {+ const sql = buildMetricQuery(+ baseQuery({ start_date: "2024-01-01", end_date: "2024-01-31", time_zone: "UTC" }),+ SITE_ID+ );+ expect(sql).toContain("AND timestamp >=");+ expect(sql).toContain("'2024-01-01'");+ });++ it("threads filters into the WHERE clause", () => {+ const filters = JSON.stringify([{ parameter: "country", type: "equals", value: ["US"] }]);+ const sql = buildMetricQuery(baseQuery({ filters }), SITE_ID);+ expect(sql).toContain("'US'");+ });+ });+});diff --git a/server/src/api/analytics/getMetric.ts b/server/src/api/analytics/getMetric.tsindex 6f11db71c..be7ec83f5 100644--- a/server/src/api/analytics/getMetric.ts+++ b/server/src/api/analytics/getMetric.ts@@ -1,10 +1,10 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { FilterParameter } from "./types.js";-import { getTimeStatement, processResults } from "./utils/utils.js";+import { getTimeStatement } from "./utils/utils.js";import { getFilterStatement, getSqlParam } from "./utils/getFilterStatement.js";import { SESSION_CHANNEL_AGG } from "./utils/sessionAttribution.js";import { FilterParams } from "@rybbit/shared";+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "./utils/analyticsQuery.js";interface GetMetricRequest {Params: {@@ -53,32 +53,18 @@ type GetMetricPaginatedResponse = {totalCount: number;};-const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boolean = false) => {- const { filters, parameter, limit, page } = request.query;- const site = request.params.siteId;+export const buildMetricQuery = (+ query: GetMetricRequest["Querystring"],+ siteId: number,+ isCountQuery: boolean = false+) => {+ const { filters, parameter } = query;- const timeStatement = getTimeStatement(request.query);+ const timeStatement = getTimeStatement(query);- const filterStatement = getFilterStatement(filters, Number(site), timeStatement);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);- let validatedLimit: number | null = null;- if (!isCountQuery && limit !== undefined) {- const parsedLimit = parseInt(String(limit), 10);- if (!isNaN(parsedLimit) && parsedLimit > 0) {- validatedLimit = parsedLimit;- }- }- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 100";-- let validatedOffset: number | null = null;- if (!isCountQuery && page !== undefined) {- const parsedPage = parseInt(String(page), 10);- if (!isNaN(parsedPage) && parsedPage >= 1) {- const pageOffset = (parsedPage - 1) * (validatedLimit || 100);- validatedOffset = pageOffset;- }- }- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);if (parameter === "event_name") {if (isCountQuery) {@@ -418,45 +404,17 @@ const getQuery = (request: FastifyRequest<GetMetricRequest>, isCountQuery: boole`;};-export async function getMetric(req: FastifyRequest<GetMetricRequest>, res: FastifyReply) {- const { parameter, page } = req.query;- const site = req.params.siteId;-- const isPaginatedRequest = page !== undefined;+export const getMetric = analyticsRoute<GetMetricRequest>(+ req => req.query.parameter,+ async (req: FastifyRequest<GetMetricRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const params = { siteId };- const dataQuery = getQuery(req, false);- const countQuery = getQuery(req, true);-- try {- // Run both queries in parallel- const [dataResult, countResult] = await Promise.all([- clickhouse.query({- query: dataQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- }),- clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- }),- ]);-- const items = await processResults<MetricItem>(dataResult);- const countData = await processResults<{ totalCount: number }>(countResult);- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;+ const result = await runPaginatedQuery<MetricItem>(+ { query: buildMetricQuery(req.query, siteId, false), params },+ { query: buildMetricQuery(req.query, siteId, true), params }+ );- return res.send({ data: { data: items, totalCount } });- } catch (error) {- console.error(`Error fetching ${parameter}:`, error);- console.error("Failed dataQuery:", dataQuery);- if (isPaginatedRequest) {- console.error("Failed countQuery:", countQuery);- }- return res.status(500).send({ error: `Failed to fetch ${parameter}` });+ return res.send({ data: result });}-}+);diff --git a/server/src/api/analytics/getOrgEventCount.ts b/server/src/api/analytics/getOrgEventCount.tsindex 5723aedc6..757bd1027 100644--- a/server/src/api/analytics/getOrgEventCount.ts+++ b/server/src/api/analytics/getOrgEventCount.ts@@ -1,8 +1,7 @@import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { getSitesUserHasAccessTo } from "../../lib/auth-utils.js";-import { processResults } from "./utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";type OrgEventCountResponse = {event_date: string;@@ -18,42 +17,27 @@ type OrgEventCountResponse = {event_count: number;}[];-export async function getOrgEventCount(- req: FastifyRequest<{- Params: {- organizationId: string;- };- Querystring: {- start_date?: string;- end_date?: string;- time_zone?: string;- };- }>,- res: FastifyReply-) {- const { organizationId } = req.params;- const { start_date, end_date, time_zone = "UTC" } = req.query;-- try {- // Get all sites the user has access to- const userSites = await getSitesUserHasAccessTo(req);-- // Filter to only sites in the requested organization- const orgSites = userSites.filter(site => site.organizationId === organizationId);-- if (orgSites.length === 0) {- return res.status(403).send({ error: "No access to organization or no sites found" });- }+interface GetOrgEventCountRequest {+ Params: {+ organizationId: string;+ };+ Querystring: {+ start_date?: string;+ end_date?: string;+ time_zone?: string;+ };+}- const siteIds = orgSites.map((site: any) => site.siteId);+export const buildOrgEventCountQuery = (query: GetOrgEventCountRequest["Querystring"], siteIds: number[]) => {+ const { start_date, end_date, time_zone = "UTC" } = query;- // Build time filter for the query- let timeFilter = "";- let fillFromDate = "";- let fillToDate = "";+ // Build time filter for the query+ let timeFilter = "";+ let fillFromDate = "";+ let fillToDate = "";- if (start_date && end_date) {- timeFilter = `AND event_hour >= toTimeZone(+ if (start_date && end_date) {+ timeFilter = `AND event_hour >= toTimeZone(toStartOfDay(toDateTime(${SqlString.escape(start_date)}, ${SqlString.escape(time_zone)})),'UTC')@@ -66,13 +50,13 @@ export async function getOrgEventCount())`;- // Set up WITH FILL parameters- fillFromDate = `FROM toTimeZone(+ // Set up WITH FILL parameters+ fillFromDate = `FROM toTimeZone(toStartOfDay(toDateTime(${SqlString.escape(start_date)}, ${SqlString.escape(time_zone)})),'UTC')`;- fillToDate = `TO if(+ fillToDate = `TO if(toDate(${SqlString.escape(end_date)}) = toDate(now(), ${SqlString.escape(time_zone)}),toStartOfDay(now()) + INTERVAL 1 DAY,toTimeZone(@@ -80,14 +64,14 @@ export async function getOrgEventCount('UTC'))`;- } else {- // No date range: return all data without WITH FILL- timeFilter = "";- fillFromDate = "";- fillToDate = "";- }+ } else {+ // No date range: return all data without WITH FILL+ timeFilter = "";+ fillFromDate = "";+ fillToDate = "";+ }- const query = `+ return `SELECTtoStartOfDay(timestamp) as event_date,countIf(type = 'pageview') as pageview_count,@@ -108,16 +92,28 @@ export async function getOrgEventCount(ORDER BY event_date${fillFromDate ? `WITH FILL ${fillFromDate} ${fillToDate} STEP INTERVAL 1 DAY` : ""}`;+};- const result = await clickhouse.query({- query,- format: "JSONEachRow",- });+export const getOrgEventCount = analyticsRoute<GetOrgEventCountRequest>(+ "organization event count",+ async (req: FastifyRequest<GetOrgEventCountRequest>, res: FastifyReply) => {+ const { organizationId } = req.params;- const data = await processResults<OrgEventCountResponse[number]>(result);+ // Get all sites the user has access to+ const userSites = await getSitesUserHasAccessTo(req);++ // Filter to only sites in the requested organization+ const orgSites = userSites.filter(site => site.organizationId === organizationId);++ if (orgSites.length === 0) {+ return res.status(403).send({ error: "No access to organization or no sites found" });+ }++ const siteIds = orgSites.map((site: any) => site.siteId);++ const data = await runAnalyticsQuery<OrgEventCountResponse[number]>({+ query: buildOrgEventCountQuery(req.query, siteIds),+ });return res.send({ data });- } catch (error) {- console.error("Error fetching organization event count:", error);- return res.status(500).send({ error: "Failed to fetch organization event count" });}-}+);diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.tsindex fbd47a855..fa83b3cfc 100644--- a/server/src/api/analytics/getOverview.ts+++ b/server/src/api/analytics/getOverview.ts@@ -1,8 +1,8 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { getFilterStatement } from "./utils/getFilterStatement.js";-import { getTimeStatement, processResults } from "./utils/utils.js";+import { getTimeStatement } from "./utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";type GetOverviewResponse = {sessions: number;@@ -13,7 +13,7 @@ type GetOverviewResponse = {session_duration: number;};-const getQuery = (params: FilterParams, siteId: number) => {+export const buildOverviewQuery = (params: FilterParams, siteId: number) => {const timeStatement = getTimeStatement(params);const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);@@ -61,46 +61,16 @@ export interface OverviewRequest {Querystring: FilterParams;}-export async function getOverview(req: FastifyRequest<OverviewRequest>, res: FastifyReply) {- const {- start_date,- end_date,- time_zone,- filters,- start_datetime,- end_datetime,- past_minutes_start,- past_minutes_end,- } = req.query;- const site = req.params.siteId;+export const getOverview = analyticsRoute<OverviewRequest>(+ "overview",+ async (req: FastifyRequest<OverviewRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);- const query = getQuery(- {- start_date,- end_date,- time_zone,- filters,- start_datetime,- end_datetime,- past_minutes_start,- past_minutes_end,- },- Number(site)- );-- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ const data = await runAnalyticsQuery<GetOverviewResponse>({+ query: buildOverviewQuery(req.query, siteId),+ params: { siteId },});- const data = await processResults<GetOverviewResponse>(result);return res.send({ data: data[0] });- } catch (error) {- console.error("Error fetching overview:", error);- return res.status(500).send({ error: "Failed to fetch overview" });}-}+);diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.tsindex 3fb8af8ac..730fb1a05 100644--- a/server/src/api/analytics/getOverviewBucketed.ts+++ b/server/src/api/analytics/getOverviewBucketed.ts@@ -1,17 +1,16 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { validateTimeStatementFillParams } from "./utils/query-validation.js";import {getTimeStatement,normalizeDatetimeForClickhouse,- processResults,TimeBucketToFn,bucketIntervalMap,} from "./utils/utils.js";import { getFilterStatement } from "./utils/getFilterStatement.js";import { TimeBucket } from "./types.js";+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);@@ -89,7 +88,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {return "";}-const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {+export const buildOverviewBucketedQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {const {start_date,end_date,@@ -183,32 +182,19 @@ ORDER BY time`;type getOverviewBucketed = { time: string; pageviews: number }[];-export async function getOverviewBucketed(- req: FastifyRequest<{- Params: {- siteId: string;- };- Querystring: FilterParams<{- bucket: TimeBucket;- }>;- }>,- res: FastifyReply-) {- const {- start_date,- end_date,- time_zone,- bucket,- filters,- start_datetime,- end_datetime,- past_minutes_start,- past_minutes_end,- } = req.query;- const site = req.params.siteId;+interface GetOverviewBucketedRequest {+ Params: {+ siteId: string;+ };+ Querystring: FilterParams<{+ bucket: TimeBucket;+ }>;+}- const query = getQuery(- {+export const getOverviewBucketed = analyticsRoute<GetOverviewBucketedRequest>(+ "pageviews",+ async (req: FastifyRequest<GetOverviewBucketedRequest>, res: FastifyReply) => {+ const {start_date,end_date,time_zone,@@ -218,23 +204,27 @@ export async function getOverviewBucketed(end_datetime,past_minutes_start,past_minutes_end,- },- Number(site)- );+ } = req.query;+ const site = req.params.siteId;- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ const data = await runAnalyticsQuery<getOverviewBucketed[number]>({+ query: buildOverviewBucketedQuery(+ {+ start_date,+ end_date,+ time_zone,+ bucket,+ filters,+ start_datetime,+ end_datetime,+ past_minutes_start,+ past_minutes_end,+ },+ Number(site)+ ),+ params: { siteId: Number(site) },});- const data = await processResults<getOverviewBucketed[number]>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching pageviews:", error);- return res.status(500).send({ error: "Failed to fetch pageviews" });}-}+);diff --git a/server/src/api/analytics/getPageTitles.ts b/server/src/api/analytics/getPageTitles.tsindex 88ddacd00..5d820c1cc 100644--- a/server/src/api/analytics/getPageTitles.ts+++ b/server/src/api/analytics/getPageTitles.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "./utils/utils.js";+import { getTimeStatement } from "./utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "./utils/getFilterStatement.js";+import { analyticsRoute, getPaginationStatements, runAnalyticsQuery, runPaginatedQuery } from "./utils/analyticsQuery.js";interface GetPageTitlesRequest {Params: {@@ -31,31 +31,18 @@ type PageTitlesPaginatedResponse = {totalCount: number;};-const getPageTitlesQuery = (request: FastifyRequest<GetPageTitlesRequest>, isCountQuery: boolean = false) => {- const { filters, limit, page } = request.query;+export const buildPageTitlesQuery = (+ query: GetPageTitlesRequest["Querystring"],+ siteId: number,+ isCountQuery: boolean = false+) => {+ const { filters } = query;- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);- let validatedLimit: number | null = null;- if (!isCountQuery && limit !== undefined) {- const parsedLimit = parseInt(String(limit), 10);- if (!isNaN(parsedLimit) && parsedLimit > 0) {- validatedLimit = parsedLimit;- }- }- // StandardSection usually shows a small number, e.g., 7 or 10. Let's default to 10 for non-paginated use.- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 10";-- let validatedOffset: number | null = null;- if (!isCountQuery && page !== undefined) {- const parsedPage = parseInt(String(page), 10);- if (!isNaN(parsedPage) && parsedPage >= 1) {- const pageOffset = (parsedPage - 1) * (validatedLimit || 10);- validatedOffset = pageOffset;- }- }- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";+ // StandardSection usually shows a small number, e.g., 7 or 10. Default to 10 for non-paginated use.+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 10, isCountQuery);// For page_title, we want to count distinct sessions that viewed this title.// We also need a representative pathname and calculate average time on page.@@ -143,47 +130,23 @@ const getPageTitlesQuery = (request: FastifyRequest<GetPageTitlesRequest>, isCou`;};-export async function getPageTitles(req: FastifyRequest<GetPageTitlesRequest>, res: FastifyReply) {- const site = req.params.siteId;- const { page } = req.query;-- const isPaginatedRequest = page !== undefined; // True if page is present+export const getPageTitles = analyticsRoute<GetPageTitlesRequest>(+ "page titles",+ async (req: FastifyRequest<GetPageTitlesRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const params = { siteId };+ const dataSpec = { query: buildPageTitlesQuery(req.query, siteId, false), params };- const dataQuery = getPageTitlesQuery(req, false);-- try {- const dataResult = await clickhouse.query({- query: dataQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- });- const items = await processResults<PageTitleItem>(dataResult);-- if (isPaginatedRequest) {- const countQuery = getPageTitlesQuery(req, true);- const countResult = await clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ if (req.query.page !== undefined) {+ const result = await runPaginatedQuery<PageTitleItem>(dataSpec, {+ query: buildPageTitlesQuery(req.query, siteId, true),+ params,});- const countData = await processResults<{ totalCount: number }>(countResult);- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;- return res.send({ data: { data: items, totalCount } });- } else {- // For non-paginated (StandardSection default) use, return the simpler structure- return res.send({ data: items });- }- } catch (error) {- console.error(`Error fetching page titles:`, error);- console.error("Failed dataQuery:", dataQuery);- if (isPaginatedRequest) {- const countQuery = getPageTitlesQuery(req, true);- console.error("Failed countQuery:", countQuery);+ return res.send({ data: result });}- return res.status(500).send({ error: `Failed to fetch page titles` });++ // For non-paginated (StandardSection default) use, return the simpler structure+ const items = await runAnalyticsQuery<PageTitleItem>(dataSpec);+ return res.send({ data: items });}-}+);diff --git a/server/src/api/analytics/getRetention.ts b/server/src/api/analytics/getRetention.tsindex 07d1a374f..9b56d39aa 100644--- a/server/src/api/analytics/getRetention.ts+++ b/server/src/api/analytics/getRetention.ts@@ -1,6 +1,5 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../db/clickhouse/clickhouse.js";-import { processResults } from "./utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js";// Define the expected shape of a single data row from the queryinterface RetentionDataRow {@@ -19,28 +18,17 @@ interface ProcessedRetentionData {range: number;}-export const getRetention = async (- req: FastifyRequest<{- Params: { siteId: string };- Querystring: { mode?: string; range?: string };- }>,- res: FastifyReply-) => {- const { siteId } = req.params;- const { mode = "week", range = "90" } = req.query; // Default to weekly mode and 90 days range-- // Validate mode parameter- const retentionMode = mode === "day" ? "day" : "week";-- // Validate range parameter (between 7-365 days)- const timeRange = Math.min(365, Math.max(7, parseInt(range) || 90));+interface GetRetentionRequest {+ Params: { siteId: string };+ Querystring: { mode?: string; range?: string };+}+export const buildRetentionQuery = (retentionMode: "day" | "week") => {// Build the appropriate SQL based on the retention modeconst periodFunction = retentionMode === "day" ? "toDate" : "toStartOfWeek";const periodDiffFunc = retentionMode === "day" ? "day" : "week";- const query = await clickhouse.query({- query: `+ return `WITH UserFirstPeriod AS (SELECT-- Use effective user ID: identified_user_id for identified users, user_id for anonymous@@ -91,26 +79,40 @@ JOIN CohortSize cs ON cr.cohort_period = cs.cohort_periodORDER BYcr.cohort_period DESC,cr.period_difference ASC;- `,- format: "JSONEachRow",- query_params: {- siteId: Number(siteId),- timeRange: timeRange,- },- });-- const results = await processResults<RetentionDataRow>(query);-- // Process data into a grid-friendly format- const processedData: ProcessedRetentionData = {- ...processRetentionData(results),- mode: retentionMode as "day" | "week",- range: timeRange,- };-- return res.send({ data: processedData });+ `;};+export const getRetention = analyticsRoute<GetRetentionRequest>(+ "retention",+ async (req: FastifyRequest<GetRetentionRequest>, res: FastifyReply) => {+ const { siteId } = req.params;+ const { mode = "week", range = "90" } = req.query; // Default to weekly mode and 90 days range++ // Validate mode parameter+ const retentionMode = mode === "day" ? "day" : "week";++ // Validate range parameter (between 7-365 days)+ const timeRange = Math.min(365, Math.max(7, parseInt(range) || 90));++ const results = await runAnalyticsQuery<RetentionDataRow>({+ query: buildRetentionQuery(retentionMode),+ params: {+ siteId: Number(siteId),+ timeRange: timeRange,+ },+ });++ // Process data into a grid-friendly format+ const processedData: ProcessedRetentionData = {+ ...processRetentionData(results),+ mode: retentionMode as "day" | "week",+ range: timeRange,+ };++ return res.send({ data: processedData });+ }+);+// Process raw retention data into a grid-friendly formatfunction processRetentionData(rawData: RetentionDataRow[]): Omit<ProcessedRetentionData, "mode" | "range"> {if (!rawData || rawData.length === 0) {diff --git a/server/src/api/analytics/goals/getGoalSessions.ts b/server/src/api/analytics/goals/getGoalSessions.tsindex b7e62312c..275309643 100644--- a/server/src/api/analytics/goals/getGoalSessions.ts+++ b/server/src/api/analytics/goals/getGoalSessions.ts@@ -1,11 +1,11 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { db } from "../../../db/postgres/postgres.js";import { goals } from "../../../db/postgres/schema.js";import { eq } from "drizzle-orm";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { GetSessionsResponse } from "../sessions/getSessions.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { buildGoalCondition } from "./goalConditions.js";export interface GetGoalSessionsRequest {@@ -19,40 +19,16 @@ export interface GetGoalSessionsRequest {}>;}-export async function getGoalSessions(req: FastifyRequest<GetGoalSessionsRequest>, res: FastifyReply) {- const { goalId, siteId } = req.params;- const { page, limit } = req.query;+export const buildGoalSessionsQuery = (+ query: GetGoalSessionsRequest["Querystring"],+ siteId: number,+ goalCondition: string+) => {+ const timeStatement = getTimeStatement(query);- try {- // Fetch the goal from PostgreSQL- const goal = await db- .select()- .from(goals)- .where(eq(goals.goalId, Number(goalId)))- .limit(1);-- if (!goal || goal.length === 0) {- return res.status(404).send({ error: "Goal not found" });- }-- const goalData = goal[0];-- // Verify the goal belongs to the site- if (goalData.siteId !== Number(siteId)) {- return res.status(403).send({ error: "Goal does not belong to this site" });- }-- const timeStatement = getTimeStatement(req.query);-- // Build the goal matching condition- const goalCondition = buildGoalCondition(goalData);- if (!goalCondition) {- return res.status(400).send({ error: "Invalid goal configuration" });- }-- // Build query to find sessions that match the goal- // First, find all session_ids that have at least one event matching the goal- const query = `+ // Build query to find sessions that match the goal+ // First, find all session_ids that have at least one event matching the goal+ return `WITH GoalSessions AS (SELECT DISTINCT session_idFROM events@@ -113,21 +89,47 @@ export async function getGoalSessions(req: FastifyRequest<GetGoalSessionsRequestFROM AggregatedSessionsLIMIT {limit:Int32} OFFSET {offset:Int32}`;+};++export const getGoalSessions = analyticsRoute<GetGoalSessionsRequest>(+ "goal sessions",+ async (req: FastifyRequest<GetGoalSessionsRequest>, res: FastifyReply) => {+ const { goalId, siteId } = req.params;+ const { page, limit } = req.query;- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ // Fetch the goal from PostgreSQL+ const goal = await db+ .select()+ .from(goals)+ .where(eq(goals.goalId, Number(goalId)))+ .limit(1);++ if (!goal || goal.length === 0) {+ return res.status(404).send({ error: "Goal not found" });+ }++ const goalData = goal[0];++ // Verify the goal belongs to the site+ if (goalData.siteId !== Number(siteId)) {+ return res.status(403).send({ error: "Goal does not belong to this site" });+ }++ // Build the goal matching condition+ const goalCondition = buildGoalCondition(goalData);+ if (!goalCondition) {+ return res.status(400).send({ error: "Invalid goal configuration" });+ }++ const data = await runAnalyticsQuery<GetSessionsResponse[number]>({+ query: buildGoalSessionsQuery(req.query, Number(siteId), goalCondition),+ params: {siteId: Number(siteId),limit: limit || 25,offset: ((page || 1) - 1) * (limit || 25),},});- const data = await processResults<GetSessionsResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching goal sessions:", error);- return res.status(500).send({ error: "Failed to fetch goal sessions" });}-}+);diff --git a/server/src/api/analytics/goals/getGoalTimeSeries.ts b/server/src/api/analytics/goals/getGoalTimeSeries.tsindex b806c37fa..d2764b1d3 100644--- a/server/src/api/analytics/goals/getGoalTimeSeries.ts+++ b/server/src/api/analytics/goals/getGoalTimeSeries.ts@@ -2,12 +2,12 @@ import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import { and, inArray, eq } from "drizzle-orm";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { db } from "../../../db/postgres/postgres.js";import { goals } from "../../../db/postgres/schema.js";import { TimeBucket } from "../types.js";import { getFilterStatement } from "../utils/getFilterStatement.js";-import { getTimeStatement, processResults, TimeBucketToFn } from "../utils/utils.js";+import { getTimeStatement, TimeBucketToFn } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { buildGoalCondition } from "./goalConditions.js";type GoalTimeSeriesPoint = {@@ -47,40 +47,26 @@ const parseGoalIds = (value: string | number[] | undefined) => {return value.split(",").map(Number).filter(Number.isFinite);};-export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesRequest>, res: FastifyReply) {- const siteId = Number(req.params.siteId);- const { bucket = "hour" } = req.query;- const timeZone = req.query.time_zone || "UTC";-- if (!TimeBucketToFn[bucket]) {- return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });- }-- const goalIds = [...new Set(parseGoalIds(req.query.goal_ids))];- if (goalIds.length === 0) {- return res.send({ data: [] });- }-- try {- const siteGoals = await db- .select()- .from(goals)- .where(and(eq(goals.siteId, siteId), inArray(goals.goalId, goalIds)));-- if (siteGoals.length === 0) {- return res.send({ data: [] });- }-- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(req.query.filters, siteId, timeStatement);- const bucketFn = TimeBucketToFn[bucket];-- const conversionQueries = siteGoals- .map(goal => {- const goalCondition = buildGoalCondition(goal);- if (!goalCondition) return null;-- return `+/**+ * Returns null when none of the site goals produces a valid conversion+ * condition — the handler responds with empty data in that case.+ */+export const buildGoalTimeSeriesQuery = (+ query: GetGoalTimeSeriesRequest["Querystring"],+ siteId: number,+ siteGoals: (typeof goals.$inferSelect)[]+): string | null => {+ const { bucket = "hour" } = query;+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);+ const bucketFn = TimeBucketToFn[bucket];++ const conversionQueries = siteGoals+ .map(goal => {+ const goalCondition = buildGoalCondition(goal);+ if (!goalCondition) return null;++ return `SELECTtoDateTime(${bucketFn}(toTimeZone(timestamp, {timeZone:String}))) AS time,${SqlString.escape(goal.goalId)} AS goal_id,@@ -93,15 +79,15 @@ export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesReq${filterStatement}GROUP BY time, goal_id`;- })- .filter(Boolean);+ })+ .filter(Boolean);- if (conversionQueries.length === 0) {- return res.send({ data: [] });- }+ if (conversionQueries.length === 0) {+ return null;+ }- const validGoalIds = siteGoals.map(goal => goal.goalId).join(", ");- const query = `+ const validGoalIds = siteGoals.map(goal => goal.goalId).join(", ");+ return `WITHsessions_by_bucket AS (SELECT@@ -131,20 +117,46 @@ export async function getGoalTimeSeries(req: FastifyRequest<GetGoalTimeSeriesReqLEFT JOIN conversions_by_goal c ON c.time = s.time AND c.goal_id = g.goal_idORDER BY s.time ASC, g.goal_id ASC`;+};++export const getGoalTimeSeries = analyticsRoute<GetGoalTimeSeriesRequest>(+ "goal time series",+ async (req: FastifyRequest<GetGoalTimeSeriesRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const { bucket = "hour" } = req.query;+ const timeZone = req.query.time_zone || "UTC";++ if (!TimeBucketToFn[bucket]) {+ return res.status(400).send({ error: `Invalid bucket value: ${bucket}` });+ }++ const goalIds = [...new Set(parseGoalIds(req.query.goal_ids))];+ if (goalIds.length === 0) {+ return res.send({ data: [] });+ }++ const siteGoals = await db+ .select()+ .from(goals)+ .where(and(eq(goals.siteId, siteId), inArray(goals.goalId, goalIds)));++ if (siteGoals.length === 0) {+ return res.send({ data: [] });+ }++ const query = buildGoalTimeSeriesQuery(req.query, siteId, siteGoals);+ if (query === null) {+ return res.send({ data: [] });+ }- const result = await clickhouse.query({+ const data = await runAnalyticsQuery<GoalTimeSeriesPoint>({query,- format: "JSONEachRow",- query_params: {+ params: {siteId,timeZone,},});- const data = await processResults<GoalTimeSeriesPoint>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching goal time series:", error);- return res.status(500).send({ error: "Failed to fetch goal time series" });}-}+);diff --git a/server/src/api/analytics/goals/getGoals.ts b/server/src/api/analytics/goals/getGoals.tsindex fc6624b49..a0410af9b 100644--- a/server/src/api/analytics/goals/getGoals.ts+++ b/server/src/api/analytics/goals/getGoals.ts@@ -1,12 +1,12 @@import { FastifyReply, FastifyRequest } from "fastify";import { db } from "../../../db/postgres/postgres.js";import { goals } from "../../../db/postgres/schema.js";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { eq, desc, asc, sql } from "drizzle-orm";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import SqlString from "sqlstring";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { buildGoalCondition } from "./goalConditions.js";// Types for the response@@ -31,36 +31,92 @@ interface GetGoalsResponse {};}-export async function getGoals(- request: FastifyRequest<{- Params: {- siteId: string;- };- Querystring: FilterParams<{- page?: string;- page_size?: string;- sort?: string;- order?: "asc" | "desc";- }>;- }>,- reply: FastifyReply-) {- const { siteId } = request.params;- const { filters, page = "1", page_size: pageSize = "10", sort = "createdAt", order = "desc" } = request.query;-- const pageNumber = parseInt(page, 10);- const pageSizeNumber = parseInt(pageSize, 10);-- // Validate page and pageSize- if (isNaN(pageNumber) || pageNumber < 1) {- return reply.status(400).send({ error: "Invalid page number" });+interface GetGoalsRequest {+ Params: {+ siteId: string;+ };+ Querystring: FilterParams<{+ page?: string;+ page_size?: string;+ sort?: string;+ order?: "asc" | "desc";+ }>;+}++export const buildGoalsTotalSessionsQuery = (query: GetGoalsRequest["Querystring"], siteId: number) => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = query.filters ? getFilterStatement(query.filters, siteId, timeStatement) : "";++ return `+ SELECT COUNT(DISTINCT session_id) AS total_sessions+ FROM events+ WHERE site_id = ${SqlString.escape(siteId)}+ ${timeStatement}+ ${filterStatement}+ `;+};++/**+ * Returns null when none of the goals produces a valid conversion condition —+ * the handler responds with zero-conversion goals in that case.+ */+export const buildGoalsConversionsQuery = (+ query: GetGoalsRequest["Querystring"],+ siteId: number,+ siteGoals: (typeof goals.$inferSelect)[]+): string | null => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = query.filters ? getFilterStatement(query.filters, siteId, timeStatement) : "";++ // Build a single query that calculates all goal conversions at once using conditional aggregation+ // This is more efficient than separate queries for each goal+ let conditionalClauses: string[] = [];++ for (const goal of siteGoals) {+ const goalCondition = buildGoalCondition(goal);+ if (!goalCondition) continue;++ conditionalClauses.push(`+ COUNT(DISTINCT IF(+ ${goalCondition},+ session_id,+ NULL+ )) AS goal_${goal.goalId}_conversions+ `);}- if (isNaN(pageSizeNumber) || pageSizeNumber < 1 || pageSizeNumber > 100) {- return reply.status(400).send({ error: "Invalid page size, must be between 1 and 100" });+ if (conditionalClauses.length === 0) {+ return null;}- try {+ return `+ SELECT+ ${conditionalClauses.join(", ")}+ FROM events+ WHERE site_id = ${SqlString.escape(siteId)}+ ${timeStatement}+ ${filterStatement}+ `;+};++export const getGoals = analyticsRoute<GetGoalsRequest>(+ "goals data",+ async (request: FastifyRequest<GetGoalsRequest>, reply: FastifyReply) => {+ const { siteId } = request.params;+ const { page = "1", page_size: pageSize = "10", sort = "createdAt", order = "desc" } = request.query;++ const pageNumber = parseInt(page, 10);+ const pageSizeNumber = parseInt(pageSize, 10);++ // Validate page and pageSize+ if (isNaN(pageNumber) || pageNumber < 1) {+ return reply.status(400).send({ error: "Invalid page number" });+ }++ if (isNaN(pageSizeNumber) || pageSizeNumber < 1 || pageSizeNumber > 100) {+ return reply.status(400).send({ error: "Invalid page size, must be between 1 and 100" });+ }+// Count total goals for pagination metadataconst totalGoalsResult = await db.select({ count: sql<number>`count(*)` })@@ -123,45 +179,15 @@ export async function getGoals(});}- // Build filter and time clauses for ClickHouse queries- const timeStatement = getTimeStatement(request.query);- const filterStatement = filters ? getFilterStatement(filters, Number(siteId), timeStatement) : "";-// First, get the total number of unique sessions (denominator for conversion rate)- const totalSessionsQuery = `- SELECT COUNT(DISTINCT session_id) AS total_sessions- FROM events- WHERE site_id = ${SqlString.escape(Number(siteId))}- ${timeStatement}- ${filterStatement}- `;-- const totalSessionsResult = await clickhouse.query({- query: totalSessionsQuery,- format: "JSONEachRow",+ const totalSessionsData = await runAnalyticsQuery<{ total_sessions: number }>({+ query: buildGoalsTotalSessionsQuery(request.query, Number(siteId)),});-- const totalSessionsData = await processResults<{ total_sessions: number }>(totalSessionsResult);const totalSessions = totalSessionsData[0]?.total_sessions || 0;- // Build a single query that calculates all goal conversions at once using conditional aggregation- // This is more efficient than separate queries for each goal- let conditionalClauses: string[] = [];-- for (const goal of siteGoals) {- const goalCondition = buildGoalCondition(goal);- if (!goalCondition) continue;+ const conversionQuery = buildGoalsConversionsQuery(request.query, Number(siteId), siteGoals);- conditionalClauses.push(`- COUNT(DISTINCT IF(- ${goalCondition},- session_id,- NULL- )) AS goal_${goal.goalId}_conversions- `);- }-- if (conditionalClauses.length === 0) {+ if (conversionQuery === null) {// If no valid goals to calculate, return the goals without conversion dataconst goalsWithZeroConversions = siteGoals.map(goal => ({...goal,@@ -182,21 +208,7 @@ export async function getGoals(}// Execute the comprehensive query- const conversionQuery = `- SELECT- ${conditionalClauses.join(", ")}- FROM events- WHERE site_id = ${SqlString.escape(Number(siteId))}- ${timeStatement}- ${filterStatement}- `;-- const conversionResult = await clickhouse.query({- query: conversionQuery,- format: "JSONEachRow",- });-- const conversionData = await processResults<Record<string, number>>(conversionResult);+ const conversionData = await runAnalyticsQuery<Record<string, number>>({ query: conversionQuery });// If we didn't get any results, use zerosconst conversions = conversionData[0] || {};@@ -223,8 +235,5 @@ export async function getGoals(totalPages,},});- } catch (error) {- console.error("Error fetching goals:", error);- return reply.status(500).send({ error: "Failed to fetch goals data" });}-}+);diff --git a/server/src/api/analytics/lite/getMetricLite.ts b/server/src/api/analytics/lite/getMetricLite.tsindex 24018753a..923565ff3 100644--- a/server/src/api/analytics/lite/getMetricLite.ts+++ b/server/src/api/analytics/lite/getMetricLite.ts@@ -1,8 +1,7 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getMetric } from "../getMetric.js";-import { processResults } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters } from "./utils.js";// Lite metric supports only dimensions backed by MVs:@@ -24,43 +23,29 @@ type LiteMetricItem = {total_count?: number;};-export async function getMetricLite(- req: FastifyRequest<{- Params: { siteId: string };- Querystring: FilterParams<{ parameter: LiteMetricParameter; limit?: number; page?: number }>;- }>,- res: FastifyReply-) {- const site = Number(req.params.siteId);- const { parameter } = req.query;- const limit = Math.min(Number(req.query.limit) || 250, 500);- const page = Math.max(Number(req.query.page) || 1, 1);- const offsetStatement = page > 1 ? `OFFSET ${(page - 1) * limit}` : "";- const filtersPresent = hasLiteFilters(req.query.filters);-- // Pull totalCount off the window column and drop it from the returned rows so- // items match the standard /metric shape.- const sendMetric = (data: LiteMetricItem[]) => {- const totalCount = data.length > 0 ? (data[0].total_count ?? data.length) : 0;- const items = data.map(({ total_count, ...rest }) => rest);- return res.send({ data: { data: items, totalCount } });- };+interface GetMetricLiteRequest {+ Params: { siteId: string };+ Querystring: FilterParams<{ parameter: LiteMetricParameter; limit?: number; page?: number }>;+}- // Filtered country/device_type lists can be served from sessions_mv_target- // (it carries both columns per session), as long as every active filter- // targets a session column. pathname isn't on the session rollup — and no- // dimensioned MV pairs pathname with another dimension — so any filtered- // pathname list falls back to the raw-events query.- if (filtersPresent) {- const filter = getLiteSessionFilter(req.query.filters);- const sessionFastPath = filter.supported && (parameter === "country" || parameter === "device_type");- if (!sessionFastPath) {- return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);- }+// Lite pagination clamps rather than validates (default 250, max 500), so it+// keeps its own logic instead of getPaginationStatements.+const getLitePagination = (query: GetMetricLiteRequest["Querystring"]) => {+ const limit = Math.min(Number(query.limit) || 250, 500);+ const page = Math.max(Number(query.page) || 1, 1);+ const offsetStatement = page > 1 ? `OFFSET ${(page - 1) * limit}` : "";+ return { limit, offsetStatement };+};- const sessionsTime = getLiteTimeStatement(req.query, "start_time");- const nonEmpty = parameter === "device_type" ? `AND ${parameter} <> ''` : "";- const query = `+// Filtered country/device_type lists can be served from sessions_mv_target+// (it carries both columns per session), as long as every active filter+// targets a session column.+export const buildMetricLiteSessionQuery = (query: GetMetricLiteRequest["Querystring"], filterSql: string) => {+ const { parameter } = query;+ const { limit, offsetStatement } = getLitePagination(query);+ const sessionsTime = getLiteTimeStatement(query, "start_time");+ const nonEmpty = parameter === "device_type" ? `AND ${parameter} <> ''` : "";+ return `SELECTvalue,pageviews,@@ -82,7 +67,7 @@ export async function getMetricLite(WHERE site_id = {siteId:Int32}${sessionsTime}${nonEmpty}- ${filter.sql}+ ${filterSql}GROUP BY session_id)GROUP BY ${parameter}@@ -91,29 +76,20 @@ export async function getMetricLite(LIMIT ${limit}${offsetStatement}`;+};- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: { siteId: site },- });- const data = await processResults<LiteMetricItem>(result);- return sendMetric(data);- } catch (error) {- console.error("Error fetching lite metric:", error);- return res.status(500).send({ error: "Failed to fetch metric" });- }- }-- const timeStatement = getLiteTimeStatement(req.query, "event_hour");+// Unfiltered lists read the dimensioned hourly MVs directly. Returns null for+// parameters lite mode doesn't support — the handler responds 400.+export const buildMetricLiteQuery = (query: GetMetricLiteRequest["Querystring"]): string | null => {+ const { parameter } = query;+ const { limit, offsetStatement } = getLitePagination(query);+ const timeStatement = getLiteTimeStatement(query, "event_hour");// Percentages are computed in an outer pass so the window function// operates on already-grouped rows. `sum(sum(...)) OVER ()` is illegal// in ClickHouse — aggregates can't nest.- let query: string;if (parameter === "pathname") {- query = `+ return `SELECTvalue,hostname,@@ -137,8 +113,9 @@ export async function getMetricLite(LIMIT ${limit}${offsetStatement}`;- } else if (parameter === "country") {- query = `+ }+ if (parameter === "country") {+ return `SELECTvalue,pageviews,@@ -160,8 +137,9 @@ export async function getMetricLite(LIMIT ${limit}${offsetStatement}`;- } else if (parameter === "device_type") {- query = `+ }+ if (parameter === "device_type") {+ return `SELECTvalue,pageviews,@@ -184,20 +162,53 @@ export async function getMetricLite(LIMIT ${limit}${offsetStatement}`;- } else {- return res.status(400).send({ error: "Lite mode does not support this parameter" });}+ return null;+};++export const getMetricLite = analyticsRoute<GetMetricLiteRequest>(+ "metric",+ async (req: FastifyRequest<GetMetricLiteRequest>, res: FastifyReply) => {+ const site = Number(req.params.siteId);+ const filtersPresent = hasLiteFilters(req.query.filters);++ // Pull totalCount off the window column and drop it from the returned rows so+ // items match the standard /metric shape.+ const sendMetric = (data: LiteMetricItem[]) => {+ const totalCount = data.length > 0 ? (data[0].total_count ?? data.length) : 0;+ const items = data.map(({ total_count, ...rest }) => rest);+ return res.send({ data: { data: items, totalCount } });+ };++ // Filtered country/device_type lists can be served from sessions_mv_target,+ // as long as every active filter targets a session column. pathname isn't+ // on the session rollup — and no dimensioned MV pairs pathname with another+ // dimension — so any filtered pathname list falls back to the raw-events+ // query.+ if (filtersPresent) {+ const filter = getLiteSessionFilter(req.query.filters);+ const sessionFastPath =+ filter.supported && (req.query.parameter === "country" || req.query.parameter === "device_type");+ if (!sessionFastPath) {+ return getMetric(req as unknown as Parameters<typeof getMetric>[0], res);+ }++ const data = await runAnalyticsQuery<LiteMetricItem>({+ query: buildMetricLiteSessionQuery(req.query, filter.sql),+ params: { siteId: site },+ });+ return sendMetric(data);+ }- try {- const result = await clickhouse.query({+ const query = buildMetricLiteQuery(req.query);+ if (query === null) {+ return res.status(400).send({ error: "Lite mode does not support this parameter" });+ }++ const data = await runAnalyticsQuery<LiteMetricItem>({query,- format: "JSONEachRow",- query_params: { siteId: site },+ params: { siteId: site },});- const data = await processResults<LiteMetricItem>(result);return sendMetric(data);- } catch (error) {- console.error("Error fetching lite metric:", error);- return res.status(500).send({ error: "Failed to fetch metric" });}-}+);diff --git a/server/src/api/analytics/lite/getOverviewBucketedLite.ts b/server/src/api/analytics/lite/getOverviewBucketedLite.tsindex 30bd8713f..48b0cee73 100644--- a/server/src/api/analytics/lite/getOverviewBucketedLite.ts+++ b/server/src/api/analytics/lite/getOverviewBucketedLite.ts@@ -1,10 +1,10 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import SqlString from "sqlstring";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getOverviewBucketed } from "../getOverviewBucketed.js";import { TimeBucket } from "../types.js";-import { TimeBucketToFn, processResults } from "../utils/utils.js";+import { TimeBucketToFn } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { getLiteFillClause, getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters, liteBucket } from "./utils.js";type GetOverviewBucketedLiteResponse = {@@ -165,71 +165,66 @@ function buildSessionMvFilteredQuery(args: {`;}-export async function getOverviewBucketedLite(- req: FastifyRequest<{- Params: { siteId: string };- Querystring: FilterParams<{ bucket: TimeBucket }>;- }>,- res: FastifyReply-) {- const site = Number(req.params.siteId);- const bucket = liteBucket(req.query.bucket);- const fn = TimeBucketToFn[bucket];- const tz = SqlString.escape(req.query.time_zone || "UTC");+interface GetOverviewBucketedLiteRequest {+ Params: { siteId: string };+ Querystring: FilterParams<{ bucket: TimeBucket }>;+}- const isAllTime =- !req.query.start_date &&- !req.query.end_date &&- req.query.past_minutes_start === undefined &&- req.query.past_minutes_end === undefined;- const fill = isAllTime ? "" : getLiteFillClause(req.query, bucket);+export const getOverviewBucketedLite = analyticsRoute<GetOverviewBucketedLiteRequest>(+ "overview",+ async (req: FastifyRequest<GetOverviewBucketedLiteRequest>, res: FastifyReply) => {+ const site = Number(req.params.siteId);+ const bucket = liteBucket(req.query.bucket);+ const fn = TimeBucketToFn[bucket];+ const tz = SqlString.escape(req.query.time_zone || "UTC");- const filtersPresent = hasLiteFilters(req.query.filters);+ const isAllTime =+ !req.query.start_date &&+ !req.query.end_date &&+ req.query.past_minutes_start === undefined &&+ req.query.past_minutes_end === undefined;+ const fill = isAllTime ? "" : getLiteFillClause(req.query, bucket);- let query: string;- if (filtersPresent) {- // Session-column filters stay on sessions_mv_target; anything else falls- // back to the raw-events query.- const filter = getLiteSessionFilter(req.query.filters);- if (!filter.supported) {- return getOverviewBucketed(req, res);- }- query = buildSessionMvFilteredQuery({- fn,- tz,- sessionsTime: getLiteTimeStatement(req.query, "start_time"),- fill,- filterSql: filter.sql,- });- } else {- const useDayBucket = bucket === "day" || bucket === "week" || bucket === "month" || bucket === "year";+ const filtersPresent = hasLiteFilters(req.query.filters);- query = useDayBucket- ? buildDayBucketQuery({- fn,- tz,- sessionTime: getLiteTimeStatement(req.query, "session_hour"),- fill,- })- : buildHourBucketQuery({- fn,- tz,- overviewTime: getLiteTimeStatement(req.query, "event_hour"),- sessionsTime: getLiteTimeStatement(req.query, "start_time"),- fill,- });- }+ let query: string;+ if (filtersPresent) {+ // Session-column filters stay on sessions_mv_target; anything else falls+ // back to the raw-events query.+ const filter = getLiteSessionFilter(req.query.filters);+ if (!filter.supported) {+ return getOverviewBucketed(req, res);+ }+ query = buildSessionMvFilteredQuery({+ fn,+ tz,+ sessionsTime: getLiteTimeStatement(req.query, "start_time"),+ fill,+ filterSql: filter.sql,+ });+ } else {+ const useDayBucket = bucket === "day" || bucket === "week" || bucket === "month" || bucket === "year";- try {- const result = await clickhouse.query({+ query = useDayBucket+ ? buildDayBucketQuery({+ fn,+ tz,+ sessionTime: getLiteTimeStatement(req.query, "session_hour"),+ fill,+ })+ : buildHourBucketQuery({+ fn,+ tz,+ overviewTime: getLiteTimeStatement(req.query, "event_hour"),+ sessionsTime: getLiteTimeStatement(req.query, "start_time"),+ fill,+ });+ }++ const data = await runAnalyticsQuery<GetOverviewBucketedLiteResponse[number]>({query,- format: "JSONEachRow",- query_params: { siteId: site },+ params: { siteId: site },});- const data = await processResults<GetOverviewBucketedLiteResponse[number]>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching lite bucketed overview:", error);- return res.status(500).send({ error: "Failed to fetch overview" });}-}+);diff --git a/server/src/api/analytics/lite/getOverviewLite.ts b/server/src/api/analytics/lite/getOverviewLite.tsindex 51eaa7a0a..77104757e 100644--- a/server/src/api/analytics/lite/getOverviewLite.ts+++ b/server/src/api/analytics/lite/getOverviewLite.ts@@ -1,8 +1,7 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getOverview } from "../getOverview.js";-import { processResults } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import { getLiteSessionFilter, getLiteTimeStatement, hasLiteFilters } from "./utils.js";type GetOverviewLiteResponse = {@@ -14,29 +13,19 @@ type GetOverviewLiteResponse = {session_duration: number;};-export async function getOverviewLite(- req: FastifyRequest<{- Params: { siteId: string };- Querystring: FilterParams;- }>,- res: FastifyReply-) {- const site = Number(req.params.siteId);- const filtersPresent = hasLiteFilters(req.query.filters);-- // Filtered queries read sessions_mv_target (one row per session) so any- // session-column filter — country, device_type, etc. — stays on the MVs.- // Filters that touch columns the session rollup doesn't carry (pathname,- // utm, …) fall back to the raw-events query.- let query: string;- if (filtersPresent) {- const filter = getLiteSessionFilter(req.query.filters);- if (!filter.supported) {- return getOverview(req, res);- }+interface GetOverviewLiteRequest {+ Params: { siteId: string };+ Querystring: FilterParams;+}- const sessionsTime = getLiteTimeStatement(req.query, "start_time");- query = `+// Filtered queries read sessions_mv_target (one row per session) so any+// session-column filter — country, device_type, etc. — stays on the MVs.+// Pass `filterSql: null` for the unfiltered single-read of the refreshable+// session_hourly_mv_target.+export const buildOverviewLiteQuery = (query: GetOverviewLiteRequest["Querystring"], filterSql: string | null) => {+ if (filterSql !== null) {+ const sessionsTime = getLiteTimeStatement(query, "start_time");+ return `SELECTsessions,pageviews,@@ -61,23 +50,24 @@ export async function getOverviewLite(FROM sessions_mv_targetWHERE site_id = {siteId:Int32}${sessionsTime}- ${filter.sql}+ ${filterSql}GROUP BY session_id))`;- } else {- const timeStatement = getLiteTimeStatement(req.query, "session_hour");+ }++ const timeStatement = getLiteTimeStatement(query, "session_hour");- // Single read of the refreshable session_hourly_mv_target — ~720 rows/month- // per site instead of millions of session rows. All 6 metrics derive from- // pre-computed sums and one HLL state.- //- // Aggregations run in the inner subquery; divisions compose plain values in- // the outer SELECT. Don't alias `sum(sessions) AS sessions` at this level —- // ClickHouse will resolve the column name to the aggregate and reject as- // nested aggregation.- query = `+ // Single read of the refreshable session_hourly_mv_target — ~720 rows/month+ // per site instead of millions of session rows. All 6 metrics derive from+ // pre-computed sums and one HLL state.+ //+ // Aggregations run in the inner subquery; divisions compose plain values in+ // the outer SELECT. Don't alias `sum(sessions) AS sessions` at this level —+ // ClickHouse will resolve the column name to the aggregate and reject as+ // nested aggregation.+ return `SELECTsessions,pageviews,@@ -97,18 +87,32 @@ export async function getOverviewLite(${timeStatement})`;- }+};++export const getOverviewLite = analyticsRoute<GetOverviewLiteRequest>(+ "overview",+ async (req: FastifyRequest<GetOverviewLiteRequest>, res: FastifyReply) => {+ const site = Number(req.params.siteId);+ const filtersPresent = hasLiteFilters(req.query.filters);++ // Filters that touch columns the session rollup doesn't carry (pathname,+ // utm, …) fall back to the raw-events query.+ let query: string;+ if (filtersPresent) {+ const filter = getLiteSessionFilter(req.query.filters);+ if (!filter.supported) {+ return getOverview(req, res);+ }+ query = buildOverviewLiteQuery(req.query, filter.sql);+ } else {+ query = buildOverviewLiteQuery(req.query, null);+ }- try {- const result = await clickhouse.query({+ const data = await runAnalyticsQuery<GetOverviewLiteResponse>({query,- format: "JSONEachRow",- query_params: { siteId: site },+ params: { siteId: site },});- const data = await processResults<GetOverviewLiteResponse>(result);+return res.send({ data: data[0] });- } catch (error) {- console.error("Error fetching lite overview:", error);- return res.status(500).send({ error: "Failed to fetch overview" });}-}+);diff --git a/server/src/api/analytics/performance/getPerformanceByDimension.ts b/server/src/api/analytics/performance/getPerformanceByDimension.tsindex bb2a47a2c..9086464cc 100644--- a/server/src/api/analytics/performance/getPerformanceByDimension.ts+++ b/server/src/api/analytics/performance/getPerformanceByDimension.ts@@ -1,8 +1,8 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, getPaginationStatements, runPaginatedQuery } from "../utils/analyticsQuery.js";interface GetPerformanceByDimensionRequest {Params: {@@ -53,9 +53,12 @@ type GetPerformanceByDimensionPaginatedResponse = {totalCount: number;};-const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isCountQuery: boolean = false) => {- const queryParams = request.query;- const { filters, limit, page, sort_by: sortBy, sort_order: sortOrder, dimension } = queryParams;+export const buildPerformanceByDimensionQuery = (+ query: GetPerformanceByDimensionRequest["Querystring"],+ siteId: number,+ isCountQuery: boolean = false+) => {+ const { filters, sort_by: sortBy, sort_order: sortOrder, dimension } = query;// Validate dimensionconst validDimensions = ["pathname", "country", "device_type", "browser", "operating_system", "region"];@@ -64,27 +67,10 @@ const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isCthrow new Error(`Invalid dimension: ${dimension}`);}- const timeStatement = getTimeStatement(request.query);- const filterStatement = getFilterStatement(filters, Number(request.params.siteId), timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);- let validatedLimit: number | null = null;- if (!isCountQuery && limit !== undefined) {- const parsedLimit = parseInt(String(limit), 10);- if (!isNaN(parsedLimit) && parsedLimit > 0) {- validatedLimit = parsedLimit;- }- }- const limitStatement = !isCountQuery && validatedLimit ? `LIMIT ${validatedLimit}` : isCountQuery ? "" : "LIMIT 100";-- let validatedOffset: number | null = null;- if (!isCountQuery && page !== undefined) {- const parsedPage = parseInt(String(page), 10);- if (!isNaN(parsedPage) && parsedPage >= 1) {- const pageOffset = (parsedPage - 1) * (validatedLimit || 100);- validatedOffset = pageOffset;- }- }- const offsetStatement = !isCountQuery && validatedOffset ? `OFFSET ${validatedOffset}` : "";+ const { limitStatement, offsetStatement } = getPaginationStatements(query, 100, isCountQuery);// Handle sortingconst validSortColumns = [@@ -206,48 +192,17 @@ const getQuery = (request: FastifyRequest<GetPerformanceByDimensionRequest>, isC`;};-export async function getPerformanceByDimension(- req: FastifyRequest<GetPerformanceByDimensionRequest>,- res: FastifyReply-) {- const { page, dimension } = req.query;- const site = req.params.siteId;-- const isPaginatedRequest = page !== undefined;+export const getPerformanceByDimension = analyticsRoute<GetPerformanceByDimensionRequest>(+ "performance by dimension",+ async (req: FastifyRequest<GetPerformanceByDimensionRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const params = { siteId };- try {- const dataQuery = getQuery(req, false);- const countQuery = getQuery(req, true);+ const result = await runPaginatedQuery<PerformanceByDimensionItem>(+ { query: buildPerformanceByDimensionQuery(req.query, siteId, false), params },+ { query: buildPerformanceByDimensionQuery(req.query, siteId, true), params }+ );- // Run both queries in parallel- const [dataResult, countResult] = await Promise.all([- clickhouse.query({- query: dataQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- }),- clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- }),- ]);-- const items = await processResults<PerformanceByDimensionItem>(dataResult);- const countData = await processResults<{ totalCount: number }>(countResult);- const totalCount = countData.length > 0 ? countData[0].totalCount : 0;-- return res.send({ data: { data: items, totalCount } });- } catch (error) {- console.error("Error fetching performance by dimension:", error);- console.error("Failed dataQuery for dimension:", dimension);- if (isPaginatedRequest) {- console.error("Failed countQuery for dimension:", dimension);- }- return res.status(500).send({ error: "Failed to fetch performance by dimension" });+ return res.send({ data: result });}-}+);diff --git a/server/src/api/analytics/performance/getPerformanceOverview.ts b/server/src/api/analytics/performance/getPerformanceOverview.tsindex 6375eb47f..875c8f15c 100644--- a/server/src/api/analytics/performance/getPerformanceOverview.ts+++ b/server/src/api/analytics/performance/getPerformanceOverview.ts@@ -1,13 +1,13 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { PerformanceOverviewMetrics } from "../types.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";-const getQuery = (params: FilterParams, siteId: number) => {- const timeStatement = getTimeStatement(params);- const filterStatement = getFilterStatement(params.filters, siteId, timeStatement);+export const buildPerformanceOverviewQuery = (query: FilterParams, siteId: number) => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);return `SELECTquantile(0.5)(lcp) AS lcp_p50,@@ -46,24 +46,16 @@ export interface PerformanceOverviewRequest {Querystring: FilterParams;}-export async function getPerformanceOverview(req: FastifyRequest<PerformanceOverviewRequest>, res: FastifyReply) {- const site = req.params.siteId;+export const getPerformanceOverview = analyticsRoute<PerformanceOverviewRequest>(+ "performance overview",+ async (req: FastifyRequest<PerformanceOverviewRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);- const query = getQuery(req.query, Number(site));-- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ const data = await runAnalyticsQuery<PerformanceOverviewMetrics>({+ query: buildPerformanceOverviewQuery(req.query, siteId),+ params: { siteId },});- const data = await processResults<PerformanceOverviewMetrics>(result);return res.send({ data: data[0] });- } catch (error) {- console.error("Error fetching performance overview:", error);- return res.status(500).send({ error: "Failed to fetch performance overview" });}-}+);diff --git a/server/src/api/analytics/performance/getPerformanceTimeSeries.ts b/server/src/api/analytics/performance/getPerformanceTimeSeries.tsindex 0b845eac2..29f36fc00 100644--- a/server/src/api/analytics/performance/getPerformanceTimeSeries.ts+++ b/server/src/api/analytics/performance/getPerformanceTimeSeries.ts@@ -1,11 +1,11 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults, TimeBucketToFn, bucketIntervalMap } from "../utils/utils.js";+import { getTimeStatement, TimeBucketToFn, bucketIntervalMap } from "../utils/utils.js";import SqlString from "sqlstring";import { validateTimeStatementFillParams } from "../utils/query-validation.js";import { TimeBucket, PerformanceTimeSeriesPoint } from "../types.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {const { params: validatedParams, bucket: validatedBucket } = validateTimeStatementFillParams(params, bucket);@@ -81,7 +81,7 @@ function getTimeStatementFill(params: FilterParams, bucket: TimeBucket) {return "";}-const getQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {+export const buildPerformanceTimeSeriesQuery = (params: FilterParams<{ bucket: TimeBucket }>, siteId: number) => {const {start_date,end_date,@@ -134,34 +134,25 @@ GROUP BY time ORDER BY time ${isAllTime ? "" : getTimeStatementFill(params, buckreturn query;};-export async function getPerformanceTimeSeries(- req: FastifyRequest<{- Params: {- siteId: string;- };- Querystring: FilterParams<{- bucket: TimeBucket;- }>;- }>,- res: FastifyReply-) {- const site = req.params.siteId;+interface GetPerformanceTimeSeriesRequest {+ Params: {+ siteId: string;+ };+ Querystring: FilterParams<{+ bucket: TimeBucket;+ }>;+}- const query = getQuery(req.query, Number(site));+export const getPerformanceTimeSeries = analyticsRoute<GetPerformanceTimeSeriesRequest>(+ "performance time series",+ async (req: FastifyRequest<GetPerformanceTimeSeriesRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },+ const data = await runAnalyticsQuery<PerformanceTimeSeriesPoint>({+ query: buildPerformanceTimeSeriesQuery(req.query, siteId),+ params: { siteId },});- const data = await processResults<PerformanceTimeSeriesPoint>(result);return res.send({ data });- } catch (error) {- console.error("Error fetching performance time series:", error);- return res.status(500).send({ error: "Failed to fetch performance time series" });}-}+);diff --git a/server/src/api/analytics/sessions/getSession.ts b/server/src/api/analytics/sessions/getSession.tsindex 08d532873..71946b3d0 100644--- a/server/src/api/analytics/sessions/getSession.ts+++ b/server/src/api/analytics/sessions/getSession.ts@@ -1,7 +1,6 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";-import { processResults } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export interface SessionDetails {session_id: string;@@ -64,11 +63,8 @@ export interface GetSessionRequest {};}-export async function getSession(req: FastifyRequest<GetSessionRequest>, res: FastifyReply) {- const { sessionId, siteId } = req.params;- const limit = req.query.limit ? parseInt(req.query.limit) : 100;- const offset = req.query.offset ? parseInt(req.query.offset) : 0;- const minutes = req.query.minutes ? parseInt(req.query.minutes) : undefined;+export const buildSessionQueries = (query: GetSessionRequest["Querystring"]) => {+ const minutes = query.minutes ? parseInt(query.minutes) : undefined;// Add time filter if minutes is providedconst timeFilter = minutes ? `timestamp > now() - interval ${minutes} minute` : "";@@ -76,9 +72,8 @@ export async function getSession(req: FastifyRequest<GetSessionRequest>, res: Fa// Add the WHERE clause connector if timeFilter existsconst timeFilterWithConnector = timeFilter ? `AND ${timeFilter}` : "";- try {- // 1. First query: Get session data derived from events- const sessionQuery = `+ // 1. First query: Get session data derived from events+ const sessionQuery = `SELECTsession_id,any(user_id) as user_id,@@ -104,7 +99,7 @@ SELECTargMaxIf(pathname, timestamp, type = 'pageview') as exit_page,any(ip) AS ipFROM events-WHERE+WHEREsite_id = {siteId:Int32}AND session_id = {sessionId:String}${timeFilterWithConnector}@@ -112,8 +107,8 @@ GROUP BY session_idLIMIT 1`;- // 2. Query to get total count of pageviews- const countQuery = `+ // 2. Query to get total count of pageviews+ const countQuery = `SELECTCOUNT(*) as totalFROM events@@ -124,8 +119,8 @@ WHERE${timeFilterWithConnector}`;- // 3. Query to get paginated pageviews- const eventsQuery = `+ // 3. Query to get paginated pageviews+ const eventsQuery = `SELECTtimestamp,pathname,@@ -147,28 +142,37 @@ LIMIT {limit:Int32}OFFSET {offset:Int32}`;+ return { sessionQuery, countQuery, eventsQuery };+};++export const getSession = analyticsRoute<GetSessionRequest>(+ "session data",+ async (req: FastifyRequest<GetSessionRequest>, res: FastifyReply) => {+ const { sessionId, siteId } = req.params;+ const limit = req.query.limit ? parseInt(req.query.limit) : 100;+ const offset = req.query.offset ? parseInt(req.query.offset) : 0;++ const { sessionQuery, countQuery, eventsQuery } = buildSessionQueries(req.query);+// Execute queries in parallel- const [sessionResultSettled, countResultSettled, eventsResultSettled] = await Promise.allSettled([- clickhouse.query({+ const [sessionData, countData, eventsData] = await Promise.all([+ runAnalyticsQuery<SessionDetails>({query: sessionQuery,- format: "JSONEachRow",- query_params: {+ params: {siteId: Number(siteId),sessionId,},}),- clickhouse.query({+ runAnalyticsQuery<{ total: number }>({query: countQuery,- format: "JSONEachRow",- query_params: {+ params: {siteId: Number(siteId),sessionId,},}),- clickhouse.query({+ runAnalyticsQuery<Event>({query: eventsQuery,- format: "JSONEachRow",- query_params: {+ params: {siteId: Number(siteId),sessionId,limit,@@ -177,25 +181,6 @@ OFFSET {offset:Int32}}),]);- // Check if queries were successful- if (sessionResultSettled.status === "rejected") {- throw sessionResultSettled.reason;- }- if (countResultSettled.status === "rejected") {- throw countResultSettled.reason;- }- if (eventsResultSettled.status === "rejected") {- throw eventsResultSettled.reason;- }-- const sessionResult = sessionResultSettled.value;- const countResult = countResultSettled.value;- const eventsResult = eventsResultSettled.value;-- const sessionData = await processResults<SessionDetails>(sessionResult);- const countData = await processResults<{ total: number }>(countResult);- const eventsData = await processResults<Event>(eventsResult);-if (!sessionData || sessionData.length === 0) {return res.status(404).send({ error: "Session not found" });}@@ -213,8 +198,5 @@ OFFSET {offset:Int32}};return res.send({ data: response });- } catch (error) {- console.error("Error fetching session data:", error);- return res.status(500).send({ error: "Failed to fetch session data" });}-}+);diff --git a/server/src/api/analytics/sessions/getSessionLocations.ts b/server/src/api/analytics/sessions/getSessionLocations.tsindex 459ecb3a5..60f27cc83 100644--- a/server/src/api/analytics/sessions/getSessionLocations.ts+++ b/server/src/api/analytics/sessions/getSessionLocations.ts@@ -1,25 +1,21 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";import { getFilterStatement } from "../utils/getFilterStatement.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";-export async function getSessionLocations(- req: FastifyRequest<{- Params: {- siteId: string;- };- Querystring: FilterParams<{}>;- }>,- res: FastifyReply-) {- const { siteId } = req.params;+export interface GetSessionLocationsRequest {+ Params: {+ siteId: string;+ };+ Querystring: FilterParams<{}>;+}- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(req.query.filters, Number(siteId), timeStatement);+export const buildSessionLocationsQuery = (query: GetSessionLocationsRequest["Querystring"], siteId: number) => {+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);- const result = await clickhouse.query({- query: `+ return `WITH stuff AS (SELECTsession_id,@@ -48,19 +44,26 @@ GROUP BYlat,lon,city,- country`,- query_params: {- site: siteId,- },- format: "JSONEachRow",- });+ country`;+};- const data = await processResults<{- lat: number;- lon: number;- count: number;- city: string;- }>(result);+export const getSessionLocations = analyticsRoute<GetSessionLocationsRequest>(+ "session locations",+ async (req: FastifyRequest<GetSessionLocationsRequest>, res: FastifyReply) => {+ const { siteId } = req.params;- return res.status(200).send({ data });-}+ const data = await runAnalyticsQuery<{+ lat: number;+ lon: number;+ count: number;+ city: string;+ }>({+ query: buildSessionLocationsQuery(req.query, Number(siteId)),+ params: {+ site: siteId,+ },+ });++ return res.status(200).send({ data });+ }+);diff --git a/server/src/api/analytics/sessions/getSessions.ts b/server/src/api/analytics/sessions/getSessions.tsindex bd078fffb..89d5772a6 100644--- a/server/src/api/analytics/sessions/getSessions.ts+++ b/server/src/api/analytics/sessions/getSessions.ts@@ -1,9 +1,9 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getFilterStatement } from "../utils/getFilterStatement.js";import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery, QuerySpec } from "../utils/analyticsQuery.js";export type GetSessionsResponse = {session_id: string;@@ -76,7 +76,7 @@ const SESSION_FIELD_MAPPINGS = {"url_parameters['utm_content']": "utm_content",};-export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res: FastifyReply) {+export const buildSessionsQuery = (query: GetSessionsRequest["Querystring"], siteId: number): QuerySpec => {const {filters,page = 1,@@ -87,26 +87,25 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:min_pageviews: minPageviewsStr,min_events: minEventsStr,min_duration: minDurationStr,- } = req.query;- const site = req.params.siteId;+ } = query;const filterIdentified = identifiedOnly === "true";const minPageviews = minPageviewsStr ? parseInt(minPageviewsStr, 10) : undefined;const minEvents = minEventsStr ? parseInt(minEventsStr, 10) : undefined;const minDuration = minDurationStr ? parseInt(minDurationStr, 10) : undefined;- const timeStatement = getTimeStatement(req.query);+ const timeStatement = getTimeStatement(query);// Use composable filter options:// - sessionLevelParams: per-event fields filter at session level (finds sessions// containing a matching event) — required for any parameter the aggregated CTE// below doesn't project, otherwise the outer WHERE hits an unknown identifier// - fieldMappings: CTE extracts UTM params as separate columns, so we need to map the field names- const filterStatement = getFilterStatement(filters, Number(site), timeStatement, {+ const filterStatement = getFilterStatement(filters, siteId, timeStatement, {sessionLevelParams: ["event_name", "pathname", "page_title", "querystring", "channel"],fieldMappings: SESSION_FIELD_MAPPINGS,});- const query = `+ const querySQL = `WITH AggregatedSessions AS (SELECTsession_id,@@ -179,31 +178,33 @@ export async function getSessions(req: FastifyRequest<GetSessionsRequest>, res:LIMIT {limit:Int32} OFFSET {offset:Int32}`;- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- user_id: userId,- session_id: sessionId,- limit: limit || 100,- offset: (page - 1) * (limit || 100),- minPageviews: minPageviews ?? 0,- minEvents: minEvents ?? 0,- minDuration: minDuration ?? 0,- },- });+ return {+ query: querySQL,+ params: {+ siteId,+ user_id: userId,+ session_id: sessionId,+ limit: limit || 100,+ offset: (page - 1) * (limit || 100),+ minPageviews: minPageviews ?? 0,+ minEvents: minEvents ?? 0,+ minDuration: minDuration ?? 0,+ },+ };+};- const data = await processResults<Omit<GetSessionsResponse[number], "traits">>(result);+export const getSessions = analyticsRoute<GetSessionsRequest>(+ "sessions",+ async (req: FastifyRequest<GetSessionsRequest>, res: FastifyReply) => {+ const site = req.params.siteId;++ const data = await runAnalyticsQuery<Omit<GetSessionsResponse[number], "traits">>(+ buildSessionsQuery(req.query, Number(site))+ );// Enrich with traits from Postgresconst dataWithTraits = await enrichWithTraits(data, Number(site));return res.send({ data: dataWithTraits });- } catch (error) {- console.error("Generated Query:", query);- console.error("Error fetching sessions:", error);- return res.status(500).send({ error: "Failed to fetch sessions" });}-}+);diff --git a/server/src/api/analytics/users/getUserInfo.ts b/server/src/api/analytics/users/getUserInfo.tsindex 51941930b..d36afe7f5 100644--- a/server/src/api/analytics/users/getUserInfo.ts+++ b/server/src/api/analytics/users/getUserInfo.ts@@ -1,12 +1,12 @@import { FilterParams } from "@rybbit/shared";import { FastifyReply, FastifyRequest } from "fastify";import { eq, and } from "drizzle-orm";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { db } from "../../../db/postgres/postgres.js";import { userProfiles, userAliases } from "../../../db/postgres/schema.js";import { getFilterStatement } from "../utils/getFilterStatement.js";import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";-import { getTimeStatement, processResults } from "../utils/utils.js";+import { getTimeStatement } from "../utils/utils.js";+import { runAnalyticsQuery } from "../utils/analyticsQuery.js";interface UserPageviewData {sessions: number;@@ -84,25 +84,11 @@ export interface UserInfoResponse {};}-export async function getUserInfo(- req: FastifyRequest<{- Params: {- siteId: string;- userId: string;- };- Querystring: FilterParams;- }>,- res: FastifyReply-) {- const { userId, siteId } = req.params;- const { filters } = req.query;-- const numericSiteId = Number(siteId);-+export const buildUserInfoQueries = (query: FilterParams, siteId: number) => {// Optional time range + dimension filters; both empty when the page is on// all-time with no filters, which keeps the original full-history behavior.- const timeStatement = getTimeStatement(req.query);- const filterStatement = getFilterStatement(filters, numericSiteId, timeStatement);+ const timeStatement = getTimeStatement(query);+ const filterStatement = getFilterStatement(query.filters, siteId, timeStatement);// Filters run in a subquery below each aggregation: the aggregate SELECTs// alias argMax(...) to the same names as raw columns (browser_version, …),@@ -118,10 +104,7 @@ export async function getUserInfo(${filterStatement}) AS events`;- try {- const [queryResult, vitalsResult, locationsResult, devicesResult, profileResult, aliasesResult] = await Promise.all([- clickhouse.query({- query: `+ const sessionsQuery = `WITH sessions AS (SELECTsession_id,@@ -190,18 +173,12 @@ export async function getUserInfo(argMaxIf(user_timezone, session_end, user_timezone != '') AS timezoneFROMsessions- `,- query_params: {- userId,- site: siteId,- },- format: "JSONEachRow",- }),- // p75 Web Vitals across every performance event this user produced.- // Separate query: the sessions CTE collapses rows per session, which- // would turn an event-level quantile into a quantile of session picks.- clickhouse.query({- query: `+ `;++ // p75 Web Vitals across every performance event this user produced.+ // Separate query: the sessions CTE collapses rows per session, which+ // would turn an event-level quantile into a quantile of session picks.+ const vitalsQuery = `SELECTquantile(0.75)(lcp) AS lcp_p75,quantile(0.75)(cls) AS cls_p75,@@ -211,17 +188,11 @@ export async function getUserInfo(COUNT(*) AS performance_eventsFROM ${scopedEvents}WHERE type = 'performance'- `,- query_params: {- userId,- site: siteId,- },- format: "JSONEachRow",- }),- // Every location this user was seen in, by session share. A session that- // moves between cities counts once per city, so shares are approximate.- clickhouse.query({- query: `+ `;++ // Every location this user was seen in, by session share. A session that+ // moves between cities counts once per city, so shares are approximate.+ const locationsQuery = `SELECTcountry,region,@@ -235,18 +206,12 @@ export async function getUserInfo(ORDER BYsessions DESC, last_seen DESCLIMIT 20- `,- query_params: {- userId,- site: siteId,- },- format: "JSONEachRow",- }),- // Every device this user was seen on. Grouped without versions so a- // browser update doesn't split one physical device into many rows;- // versions and screen are argMax'd to the latest sighting instead.- clickhouse.query({- query: `+ `;++ // Every device this user was seen on. Grouped without versions so a+ // browser update doesn't split one physical device into many rows;+ // versions and screen are argMax'd to the latest sighting instead.+ const devicesQuery = `SELECTdevice_type,browser,@@ -264,13 +229,38 @@ export async function getUserInfo(ORDER BYsessions DESC, last_seen DESCLIMIT 20- `,- query_params: {- userId,- site: siteId,- },- format: "JSONEachRow",- }),+ `;++ return { sessionsQuery, vitalsQuery, locationsQuery, devicesQuery };+};++export async function getUserInfo(+ req: FastifyRequest<{+ Params: {+ siteId: string;+ userId: string;+ };+ Querystring: FilterParams;+ }>,+ res: FastifyReply+) {+ const { userId, siteId } = req.params;++ const numericSiteId = Number(siteId);++ const { sessionsQuery, vitalsQuery, locationsQuery, devicesQuery } = buildUserInfoQueries(req.query, numericSiteId);++ const chParams = {+ userId,+ site: siteId,+ };++ try {+ const [data, vitalsData, locations, devices, profileResult, aliasesResult] = await Promise.all([+ runAnalyticsQuery<UserPageviewData>({ query: sessionsQuery, params: chParams }),+ runAnalyticsQuery<UserVitalsData>({ query: vitalsQuery, params: chParams }),+ runAnalyticsQuery<UserLocationBreakdown>({ query: locationsQuery, params: chParams }),+ runAnalyticsQuery<UserDeviceBreakdown>({ query: devicesQuery, params: chParams }),// Get user profile traits from Postgresdb.select()@@ -287,11 +277,6 @@ export async function getUserInfo(.where(and(eq(userAliases.siteId, numericSiteId), eq(userAliases.userId, userId))),]);- const data = await processResults<UserPageviewData>(queryResult);- const vitalsData = await processResults<UserVitalsData>(vitalsResult);- const locations = await processResults<UserLocationBreakdown>(locationsResult);- const devices = await processResults<UserDeviceBreakdown>(devicesResult);-// If no data found for userif (data.length === 0) {return res.status(404).send({diff --git a/server/src/api/analytics/users/getUserSessionCount.ts b/server/src/api/analytics/users/getUserSessionCount.tsindex 5e90f2e85..e531d1270 100644--- a/server/src/api/analytics/users/getUserSessionCount.ts+++ b/server/src/api/analytics/users/getUserSessionCount.ts@@ -1,7 +1,6 @@import { FastifyReply, FastifyRequest } from "fastify";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { getFilterStatement } from "../utils/getFilterStatement.js";-import { processResults } from "../utils/utils.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";import SqlString from "sqlstring";export interface GetUserSessionCountRequest {@@ -20,19 +19,14 @@ export type GetUserSessionCountResponse = {sessions: number;}[];-export async function getUserSessionCount(req: FastifyRequest<GetUserSessionCountRequest>, res: FastifyReply) {- const { siteId } = req.params;- const { user_id: userId, time_zone: timeZone = "UTC", filters } = req.query;-- if (!userId) {- return res.status(400).send({ error: "user_id is required" });- }+export const buildUserSessionCountQuery = (query: GetUserSessionCountRequest["Querystring"], siteId: number) => {+ const { time_zone: timeZone = "UTC", filters } = query;// The calendar spans the user's full history, so dimension filters apply// but no time range does.- const filterStatement = getFilterStatement(filters ?? "", Number(siteId));+ const filterStatement = getFilterStatement(filters ?? "", siteId);- const query = `+ return `SELECTtoDate(timestamp, ${SqlString.escape(timeZone)}) as date,count(DISTINCT session_id) as sessions@@ -44,24 +38,28 @@ export async function getUserSessionCount(req: FastifyRequest<GetUserSessionCounGROUP BY dateORDER BY date ASC`;+};++export const getUserSessionCount = analyticsRoute<GetUserSessionCountRequest>(+ "user session count",+ async (req: FastifyRequest<GetUserSessionCountRequest>, res: FastifyReply) => {+ const { siteId } = req.params;+ const { user_id: userId } = req.query;- try {- const result = await clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ if (!userId) {+ return res.status(400).send({ error: "user_id is required" });+ }++ const data = await runAnalyticsQuery<GetUserSessionCountResponse[number]>({+ query: buildUserSessionCountQuery(req.query, Number(siteId)),+ params: {siteId: Number(siteId),userId,},});- const data = await processResults<GetUserSessionCountResponse[number]>(result);-return res.send({data,});- } catch (error) {- console.error("Error fetching user session count:", error);- return res.status(500).send({ error: "Failed to fetch user session count" });}-}+);diff --git a/server/src/api/analytics/users/getUserTraits.ts b/server/src/api/analytics/users/getUserTraits.tsindex 008506046..4ee875274 100644--- a/server/src/api/analytics/users/getUserTraits.ts+++ b/server/src/api/analytics/users/getUserTraits.ts@@ -1,18 +1,27 @@import { FastifyReply, FastifyRequest } from "fastify";import { sql } from "drizzle-orm";import { db } from "../../../db/postgres/postgres.js";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { processResults } from "../utils/utils.js";--export async function getUserTraitKeys(- req: FastifyRequest<{- Params: { siteId: string };- }>,- res: FastifyReply-) {- const siteId = Number(req.params.siteId);-- try {+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";++export interface GetUserTraitKeysRequest {+ Params: { siteId: string };+}++export interface GetUserTraitValuesRequest {+ Params: { siteId: string };+ Querystring: { key: string; limit?: string; offset?: string };+}++export interface GetUserTraitValueUsersRequest {+ Params: { siteId: string };+ Querystring: { key: string; value: string; limit?: string; offset?: string };+}++export const getUserTraitKeys = analyticsRoute<GetUserTraitKeysRequest>(+ "user trait keys",+ async (req: FastifyRequest<GetUserTraitKeysRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+const result = await db.execute<{ key: string; user_count: number }>(sql`SELECT key, COUNT(*)::int AS user_countFROM user_profiles,@@ -28,30 +37,22 @@ export async function getUserTraitKeys(userCount: row.user_count,})),});- } catch (error) {- console.error("Error fetching user trait keys:", error);- return res.status(500).send({ error: "Failed to fetch user trait keys" });}-}+);-export async function getUserTraitValues(- req: FastifyRequest<{- Params: { siteId: string };- Querystring: { key: string; limit?: string; offset?: string };- }>,- res: FastifyReply-) {- const siteId = Number(req.params.siteId);- const { key, limit: limitStr = "1000", offset: offsetStr = "0" } = req.query;-- if (!key) {- return res.status(400).send({ error: "key query parameter is required" });- }+export const getUserTraitValues = analyticsRoute<GetUserTraitValuesRequest>(+ "user trait values",+ async (req: FastifyRequest<GetUserTraitValuesRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const { key, limit: limitStr = "1000", offset: offsetStr = "0" } = req.query;++ if (!key) {+ return res.status(400).send({ error: "key query parameter is required" });+ }- const limit = parseInt(limitStr, 10);- const offset = parseInt(offsetStr, 10);+ const limit = parseInt(limitStr, 10);+ const offset = parseInt(offsetStr, 10);- try {const [valuesResult, countResult] = await Promise.all([db.execute<{ value: string; user_count: number }>(sql`SELECT traits->>${key} AS value, COUNT(*)::int AS user_count@@ -78,30 +79,42 @@ export async function getUserTraitValues(total,hasMore: offset + limit < total,});- } catch (error) {- console.error("Error fetching user trait values:", error);- return res.status(500).send({ error: "Failed to fetch user trait values" });}-}+);-export async function getUserTraitValueUsers(- req: FastifyRequest<{- Params: { siteId: string };- Querystring: { key: string; value: string; limit?: string; offset?: string };- }>,- res: FastifyReply-) {- const siteId = Number(req.params.siteId);- const { key, value, limit: limitStr = "50", offset: offsetStr = "0" } = req.query;-- if (!key || value === undefined) {- return res.status(400).send({ error: "key and value query parameters are required" });- }+// Session counts + metadata from ClickHouse for a set of identified users.+// Use events.identified_user_id to avoid conflict with the argMax alias+export const buildTraitValueUsersQuery = () => `+ SELECT+ COALESCE(NULLIF(events.identified_user_id, ''), events.user_id) AS effective_user_id,+ argMax(events.user_id, timestamp) AS user_id,+ argMax(events.identified_user_id, timestamp) AS identified_user_id,+ argMax(country, timestamp) AS country,+ argMax(region, timestamp) AS region,+ argMax(city, timestamp) AS city,+ argMax(browser, timestamp) AS browser,+ argMax(operating_system, timestamp) AS operating_system,+ argMax(device_type, timestamp) AS device_type,+ count(DISTINCT session_id) AS sessions+ FROM events+ WHERE site_id = {siteId:Int32}+ AND events.identified_user_id IN ({userIds:Array(String)})+ GROUP BY effective_user_id+ `;- const limit = parseInt(limitStr, 10);- const offset = parseInt(offsetStr, 10);+export const getUserTraitValueUsers = analyticsRoute<GetUserTraitValueUsersRequest>(+ "trait value users",+ async (req: FastifyRequest<GetUserTraitValueUsersRequest>, res: FastifyReply) => {+ const siteId = Number(req.params.siteId);+ const { key, value, limit: limitStr = "50", offset: offsetStr = "0" } = req.query;++ if (!key || value === undefined) {+ return res.status(400).send({ error: "key and value query parameters are required" });+ }++ const limit = parseInt(limitStr, 10);+ const offset = parseInt(offsetStr, 10);- try {// Step 1: Find matching user IDs and their traits from Postgresconst [profilesResult, countResult] = await Promise.all([db.execute<{ user_id: string; traits: Record<string, unknown> | null }>(sql`@@ -128,35 +141,7 @@ export async function getUserTraitValueUsers(const userIds = profiles.map((p: any) => p.user_id);// Step 2: Get session counts + metadata from ClickHouse- // Use events.identified_user_id to avoid conflict with the argMax alias- const chQuery = `- SELECT- COALESCE(NULLIF(events.identified_user_id, ''), events.user_id) AS effective_user_id,- argMax(events.user_id, timestamp) AS user_id,- argMax(events.identified_user_id, timestamp) AS identified_user_id,- argMax(country, timestamp) AS country,- argMax(region, timestamp) AS region,- argMax(city, timestamp) AS city,- argMax(browser, timestamp) AS browser,- argMax(operating_system, timestamp) AS operating_system,- argMax(device_type, timestamp) AS device_type,- count(DISTINCT session_id) AS sessions- FROM events- WHERE site_id = {siteId:Int32}- AND events.identified_user_id IN ({userIds:Array(String)})- GROUP BY effective_user_id- `;-- const chResult = await clickhouse.query({- query: chQuery,- format: "JSONEachRow",- query_params: {- siteId,- userIds,- },- });-- const chData = await processResults<{+ const chData = await runAnalyticsQuery<{effective_user_id: string;user_id: string;identified_user_id: string;@@ -167,7 +152,13 @@ export async function getUserTraitValueUsers(operating_system: string;device_type: string;sessions: number;- }>(chResult);+ }>({+ query: buildTraitValueUsersQuery(),+ params: {+ siteId,+ userIds,+ },+ });// Step 3: Build lookup from ClickHouse data keyed by identified_user_id,// then iterate Postgres profiles so every profile is returned even if@@ -197,8 +188,5 @@ export async function getUserTraitValueUsers(total,hasMore: offset + limit < total,});- } catch (error) {- console.error("Error fetching trait value users:", error);- return res.status(500).send({ error: "Failed to fetch trait value users" });}-}+);diff --git a/server/src/api/analytics/users/getUsers.ts b/server/src/api/analytics/users/getUsers.tsindex ecc814f38..25b9841f5 100644--- a/server/src/api/analytics/users/getUsers.ts+++ b/server/src/api/analytics/users/getUsers.ts@@ -1,11 +1,11 @@import { FastifyReply, FastifyRequest } from "fastify";import { sql, SQL } from "drizzle-orm";-import { clickhouse } from "../../../db/clickhouse/clickhouse.js";import { db } from "../../../db/postgres/postgres.js";-import { enrichWithTraits, getTimeStatement, processResults } from "../utils/utils.js";+import { enrichWithTraits, getTimeStatement } from "../utils/utils.js";import { FilterParams } from "@rybbit/shared";import { getFilterStatement } from "../utils/getFilterStatement.js";import { SESSION_CHANNEL_AGG, SESSION_REFERRER_AGG } from "../utils/sessionAttribution.js";+import { analyticsRoute, runAnalyticsQuery } from "../utils/analyticsQuery.js";export type GetUsersResponse = {user_id: string; // Device fingerprint@@ -41,56 +41,21 @@ export interface GetUsersRequest {}>;}-export async function getUsers(req: FastifyRequest<GetUsersRequest>, res: FastifyReply) {+export const buildUsersQuery = (+ query: GetUsersRequest["Querystring"],+ siteId: number,+ matchingUserIds: string[] | null,+ isCountQuery: boolean = false+) => {const {filters,- page = "1",- page_size: pageSize = "100",sort_by: sortBy = "last_seen",sort_order: sortOrder = "desc",identified_only: identifiedOnly = "false",- search,- search_field: searchField = "username",- } = req.query;- const site = req.params.siteId;- let filterIdentified = identifiedOnly === "true";-- // Search for matching user IDs in Postgres when search is provided- const MAX_MATCHING_USER_IDS = 10000;- let matchingUserIds: string[] | null = null;- if (search && search.trim()) {- const searchTerm = `%${search.trim()}%`;- const siteId = Number(site);-- const fieldConditions: Record<string, SQL> = {- username: sql`traits->>'username' ILIKE ${searchTerm}`,- name: sql`traits->>'name' ILIKE ${searchTerm}`,- email: sql`traits->>'email' ILIKE ${searchTerm}`,- user_id: sql`user_id ILIKE ${searchTerm}`,- };- const condition = fieldConditions[searchField] ?? fieldConditions.username;-- const searchResult = await db.execute<{ user_id: string }>(sql`- SELECT user_id FROM user_profiles- WHERE site_id = ${siteId} AND ${condition}- LIMIT ${MAX_MATCHING_USER_IDS}- `);-- matchingUserIds = searchResult.map((r) => r.user_id);- if (matchingUserIds.length === 0) {- return res.send({- data: [],- totalCount: 0,- page: parseInt(page, 10),- pageSize: parseInt(pageSize, 10),- });- }- filterIdentified = true;- }-- const pageNum = parseInt(page, 10);- const pageSizeNum = parseInt(pageSize, 10);- const offset = (pageNum - 1) * pageSizeNum;+ } = query;+ // Search results force the identified-only view: matching user IDs come from+ // Postgres profiles, which only exist for identified users.+ const filterIdentified = identifiedOnly === "true" || matchingUserIds !== null;// Validate sort parametersconst validSortFields = ["first_seen", "last_seen", "pageviews", "sessions", "events"];@@ -98,18 +63,46 @@ export async function getUsers(req: FastifyRequest<GetUsersRequest>, res: Fastifconst actualSortOrder = sortOrder === "asc" ? "ASC" : "DESC";// Generate filter statement and time statement- const timeStatement = getTimeStatement(req.query);+ const timeStatement = getTimeStatement(query);// Applied against raw events (same placement as the count queries): the// aggregate doesn't project every filterable column (pathname, querystring,// utm_*, …), and event-level placement keeps the returned rows consistent// with totalCount.- const filterStatement = getFilterStatement(filters, Number(site), timeStatement);+ const filterStatement = getFilterStatement(filters, siteId, timeStatement);++ // Query to get total count+ if (isCountQuery) {+ return filterIdentified+ ? `+SELECT count(*) AS total_count+FROM (+ SELECT DISTINCT identified_user_id+ FROM events+ WHERE+ site_id = {siteId:Int32}+ AND identified_user_id != ''+ ${timeStatement}+ ${filterStatement}+ ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}+)+`+ : `+SELECT+ count(DISTINCT COALESCE(NULLIF(events.identified_user_id, ''), events.user_id)) AS total_count+FROM events+WHERE+ site_id = {siteId:Int32}+ ${filterStatement}+ ${timeStatement}+ ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}+ `;+ }// Filters must run in a subquery below the aggregation: the aggregate SELECT// aliases argMax(...) to the same names as the raw columns (country, browser,// …), and ClickHouse resolves unqualified WHERE references at that level to// the aliases, throwing ILLEGAL_AGGREGATION.- const query = `+ return `WITH AggregatedUsers AS (SELECT-- Group by effective user: identified_user_id for identified users, user_id (device) for anonymous@@ -156,58 +149,70 @@ ${filterIdentified ? "AND identified_user_id != ''" : ""}ORDER BY ${actualSortBy} ${actualSortOrder}LIMIT {limit:Int32} OFFSET {offset:Int32}`;+};++export const getUsers = analyticsRoute<GetUsersRequest>(+ "users",+ async (req: FastifyRequest<GetUsersRequest>, res: FastifyReply) => {+ const { page = "1", page_size: pageSize = "100", search, search_field: searchField = "username" } = req.query;+ const site = req.params.siteId;++ // Search for matching user IDs in Postgres when search is provided+ const MAX_MATCHING_USER_IDS = 10000;+ let matchingUserIds: string[] | null = null;+ if (search && search.trim()) {+ const searchTerm = `%${search.trim()}%`;+ const siteId = Number(site);++ const fieldConditions: Record<string, SQL> = {+ username: sql`traits->>'username' ILIKE ${searchTerm}`,+ name: sql`traits->>'name' ILIKE ${searchTerm}`,+ email: sql`traits->>'email' ILIKE ${searchTerm}`,+ user_id: sql`user_id ILIKE ${searchTerm}`,+ };+ const condition = fieldConditions[searchField] ?? fieldConditions.username;++ const searchResult = await db.execute<{ user_id: string }>(sql`+ SELECT user_id FROM user_profiles+ WHERE site_id = ${siteId} AND ${condition}+ LIMIT ${MAX_MATCHING_USER_IDS}+ `);++ matchingUserIds = searchResult.map((r) => r.user_id);+ if (matchingUserIds.length === 0) {+ return res.send({+ data: [],+ totalCount: 0,+ page: parseInt(page, 10),+ pageSize: parseInt(pageSize, 10),+ });+ }+ }- // Query to get total count- const countQuery = filterIdentified- ? `-SELECT count(*) AS total_count-FROM (- SELECT DISTINCT identified_user_id- FROM events- WHERE- site_id = {siteId:Int32}- AND identified_user_id != ''- ${timeStatement}- ${filterStatement}- ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}-)-`- : `-SELECT- count(DISTINCT COALESCE(NULLIF(events.identified_user_id, ''), events.user_id)) AS total_count-FROM events-WHERE- site_id = {siteId:Int32}- ${filterStatement}- ${timeStatement}- ${matchingUserIds ? "AND events.identified_user_id IN ({matchingUserIds:Array(String)})" : ""}- `;+ const pageNum = parseInt(page, 10);+ const pageSizeNum = parseInt(pageSize, 10);+ const offset = (pageNum - 1) * pageSizeNum;- try {// Execute both queries in parallel- const [result, countResult] = await Promise.all([- clickhouse.query({- query,- format: "JSONEachRow",- query_params: {+ const [data, countData] = await Promise.all([+ runAnalyticsQuery<Omit<GetUsersResponse[number], "traits">>({+ query: buildUsersQuery(req.query, Number(site), matchingUserIds, false),+ params: {siteId: Number(site),limit: pageSizeNum,offset,...(matchingUserIds ? { matchingUserIds } : {}),},}),- clickhouse.query({- query: countQuery,- format: "JSONEachRow",- query_params: {+ runAnalyticsQuery<{ total_count: number }>({+ query: buildUsersQuery(req.query, Number(site), matchingUserIds, true),+ params: {siteId: Number(site),...(matchingUserIds ? { matchingUserIds } : {}),},}),]);- const data = await processResults<Omit<GetUsersResponse[number], "traits">>(result);- const countData = await processResults<{ total_count: number }>(countResult);const totalCount = countData[0]?.total_count || 0;// Enrich with traits from Postgres@@ -219,8 +224,5 @@ WHEREpage: pageNum,pageSize: pageSizeNum,});- } catch (error) {- console.error("Error fetching users:", error);- return res.status(500).send({ error: "Failed to fetch users" });}-}+);diff --git a/server/src/api/analytics/utils/analyticsQuery.test.ts b/server/src/api/analytics/utils/analyticsQuery.test.tsnew file mode 100644index 000000000..42ff966e8--- /dev/null+++ b/server/src/api/analytics/utils/analyticsQuery.test.ts@@ -0,0 +1,161 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++vi.mock("../../../db/clickhouse/clickhouse.js", () => ({+ clickhouse: { query: vi.fn() },+}));+vi.mock("../../../db/postgres/postgres.js", () => ({+ db: {},+}));++import { clickhouse } from "../../../db/clickhouse/clickhouse.js";+import {+ AnalyticsQueryError,+ analyticsRoute,+ getPaginationStatements,+ runAnalyticsQuery,+ runPaginatedQuery,+} from "./analyticsQuery.js";++const mockQuery = vi.mocked(clickhouse.query);++const resultSet = (rows: unknown[]) => ({ json: async () => rows }) as any;++const mockRes = () => {+ const res: any = {};+ res.status = vi.fn(() => res);+ res.send = vi.fn(() => res);+ return res;+};++beforeEach(() => {+ vi.clearAllMocks();+});++describe("getPaginationStatements", () => {+ it("falls back to the default limit", () => {+ expect(getPaginationStatements({}, 100)).toEqual({ limitStatement: "LIMIT 100", offsetStatement: "" });+ });++ it("uses a valid explicit limit, numeric or string", () => {+ expect(getPaginationStatements({ limit: 25 }, 100).limitStatement).toBe("LIMIT 25");+ expect(getPaginationStatements({ limit: "25" }, 100).limitStatement).toBe("LIMIT 25");+ });++ it("rejects invalid limits", () => {+ expect(getPaginationStatements({ limit: "abc" }, 100).limitStatement).toBe("LIMIT 100");+ expect(getPaginationStatements({ limit: -5 }, 100).limitStatement).toBe("LIMIT 100");+ expect(getPaginationStatements({ limit: 0 }, 100).limitStatement).toBe("LIMIT 100");+ });++ it("computes the offset from page and limit", () => {+ expect(getPaginationStatements({ limit: 25, page: 3 }, 100).offsetStatement).toBe("OFFSET 50");+ expect(getPaginationStatements({ page: 2 }, 10).offsetStatement).toBe("OFFSET 10");+ });++ it("emits no OFFSET for page 1 or invalid pages", () => {+ expect(getPaginationStatements({ page: 1 }, 100).offsetStatement).toBe("");+ expect(getPaginationStatements({ page: 0 }, 100).offsetStatement).toBe("");+ expect(getPaginationStatements({ page: "abc" }, 100).offsetStatement).toBe("");+ });++ it("emits no clauses for count queries", () => {+ expect(getPaginationStatements({ limit: 25, page: 3 }, 100, true)).toEqual({+ limitStatement: "",+ offsetStatement: "",+ });+ });+});++describe("runAnalyticsQuery", () => {+ it("executes the spec and coerces numeric strings", async () => {+ mockQuery.mockResolvedValueOnce(resultSet([{ value: "Chrome", count: "42" }]));++ const rows = await runAnalyticsQuery<{ value: string; count: number }>({+ query: "SELECT 1",+ params: { siteId: 1 },+ });++ expect(mockQuery).toHaveBeenCalledWith({+ query: "SELECT 1",+ format: "JSONEachRow",+ query_params: { siteId: 1 },+ });+ expect(rows).toEqual([{ value: "Chrome", count: 42 }]);+ });++ it("wraps failures in AnalyticsQueryError carrying the SQL", async () => {+ const boom = new Error("boom");+ mockQuery.mockRejectedValueOnce(boom);++ const promise = runAnalyticsQuery({ query: "SELECT bad" });++ await expect(promise).rejects.toBeInstanceOf(AnalyticsQueryError);+ await promise.catch((error: AnalyticsQueryError) => {+ expect(error.original).toBe(boom);+ expect(error.queries).toEqual(["SELECT bad"]);+ });+ });+});++describe("runPaginatedQuery", () => {+ it("runs data and count queries and combines them", async () => {+ mockQuery+ .mockResolvedValueOnce(resultSet([{ value: "a" }, { value: "b" }]))+ .mockResolvedValueOnce(resultSet([{ totalCount: "17" }]));++ const result = await runPaginatedQuery({ query: "SELECT data" }, { query: "SELECT count" });++ expect(result).toEqual({ data: [{ value: "a" }, { value: "b" }], totalCount: 17 });+ });++ it("defaults totalCount to 0 when the count query returns no rows", async () => {+ mockQuery.mockResolvedValueOnce(resultSet([])).mockResolvedValueOnce(resultSet([]));++ const result = await runPaginatedQuery({ query: "SELECT data" }, { query: "SELECT count" });++ expect(result).toEqual({ data: [], totalCount: 0 });+ });+});++describe("analyticsRoute", () => {+ it("passes successful responses through", async () => {+ const res = mockRes();+ const handler = analyticsRoute("things", async (_req, reply) => reply.send({ data: [1] }));++ await handler({} as any, res);++ expect(res.send).toHaveBeenCalledWith({ data: [1] });+ expect(res.status).not.toHaveBeenCalled();+ });++ it("maps errors to a 500 with the label", async () => {+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});+ const res = mockRes();+ const handler = analyticsRoute("things", async () => {+ throw new Error("boom");+ });++ await handler({} as any, res);++ expect(res.status).toHaveBeenCalledWith(500);+ expect(res.send).toHaveBeenCalledWith({ error: "Failed to fetch things" });+ consoleError.mockRestore();+ });++ it("derives the label from the request and logs failed SQL", async () => {+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});+ const res = mockRes();+ const handler = analyticsRoute<{ Querystring: { parameter: string } }>(+ req => req.query.parameter,+ async () => {+ throw new AnalyticsQueryError(new Error("boom"), ["SELECT bad"]);+ }+ );++ await handler({ query: { parameter: "browser" } } as any, res);++ expect(res.send).toHaveBeenCalledWith({ error: "Failed to fetch browser" });+ expect(consoleError).toHaveBeenCalledWith("Failed query:", "SELECT bad");+ consoleError.mockRestore();+ });+});diff --git a/server/src/api/analytics/utils/analyticsQuery.ts b/server/src/api/analytics/utils/analyticsQuery.tsnew file mode 100644index 000000000..341b10822--- /dev/null+++ b/server/src/api/analytics/utils/analyticsQuery.ts@@ -0,0 +1,117 @@+import { FastifyReply, FastifyRequest, RouteGenericInterface } from "fastify";+import { clickhouse } from "../../../db/clickhouse/clickhouse.js";+import { processResults } from "./utils.js";++/**+ * A fully-built ClickHouse query plus its bound parameters. Handlers build+ * specs with pure functions (testable without Fastify or a live database);+ * only the executor below ever talks to ClickHouse.+ */+export interface QuerySpec {+ query: string;+ params?: Record<string, unknown>;+}++/** Carries the failed SQL so the route wrapper can log it alongside the cause. */+export class AnalyticsQueryError extends Error {+ constructor(+ readonly original: unknown,+ readonly queries: string[]+ ) {+ super("ClickHouse query failed");+ this.name = "AnalyticsQueryError";+ }+}++export async function runAnalyticsQuery<T>(spec: QuerySpec): Promise<T[]> {+ try {+ const result = await clickhouse.query({+ query: spec.query,+ format: "JSONEachRow",+ query_params: spec.params,+ });+ return await processResults<T>(result);+ } catch (error) {+ throw new AnalyticsQueryError(error, [spec.query]);+ }+}++export async function runPaginatedQuery<T>(+ dataSpec: QuerySpec,+ countSpec: QuerySpec+): Promise<{ data: T[]; totalCount: number }> {+ const [data, countRows] = await Promise.all([+ runAnalyticsQuery<T>(dataSpec),+ runAnalyticsQuery<{ totalCount: number }>(countSpec),+ ]);+ return { data, totalCount: countRows[0]?.totalCount ?? 0 };+}++/**+ * LIMIT/OFFSET clauses from untrusted `limit`/`page` query params.+ * Count queries get no clauses; invalid or missing values fall back to+ * `defaultLimit`; page 1 (offset 0) emits no OFFSET clause.+ */+export function getPaginationStatements(+ { limit, page }: { limit?: number | string; page?: number | string },+ defaultLimit: number,+ isCountQuery = false+): { limitStatement: string; offsetStatement: string } {+ if (isCountQuery) {+ return { limitStatement: "", offsetStatement: "" };+ }++ let validatedLimit: number | null = null;+ if (limit !== undefined) {+ const parsedLimit = parseInt(String(limit), 10);+ if (!isNaN(parsedLimit) && parsedLimit > 0) {+ validatedLimit = parsedLimit;+ }+ }+ const limitStatement = `LIMIT ${validatedLimit ?? defaultLimit}`;++ let offsetStatement = "";+ if (page !== undefined) {+ const parsedPage = parseInt(String(page), 10);+ if (!isNaN(parsedPage) && parsedPage >= 1) {+ const offset = (parsedPage - 1) * (validatedLimit ?? defaultLimit);+ if (offset > 0) {+ offsetStatement = `OFFSET ${offset}`;+ }+ }+ }++ return { limitStatement, offsetStatement };+}++type AnalyticsHandler<R extends RouteGenericInterface> = (+ req: FastifyRequest<R>,+ res: FastifyReply+) => Promise<unknown>;++/**+ * Wraps a handler with the standard analytics error policy: failures log the+ * label (and the failed SQL when available) and respond 500 with+ * `Failed to fetch <label>`. The label may derive from the request.+ */+export function analyticsRoute<R extends RouteGenericInterface>(+ errorLabel: string | ((req: FastifyRequest<R>) => string),+ handler: AnalyticsHandler<R>+): AnalyticsHandler<R> {+ return async (req, res) => {+ try {+ return await handler(req, res);+ } catch (error) {+ const label = typeof errorLabel === "function" ? errorLabel(req) : errorLabel;+ if (error instanceof AnalyticsQueryError) {+ console.error(`Error fetching ${label}:`, error.original);+ for (const query of error.queries) {+ console.error("Failed query:", query);+ }+ } else {+ console.error(`Error fetching ${label}:`, error);+ }+ return res.status(500).send({ error: `Failed to fetch ${label}` });+ }+ };+}diff --git a/server/src/api/analytics/utils/utils.ts b/server/src/api/analytics/utils/utils.tsindex 2a08a2c41..23f95f1b3 100644--- a/server/src/api/analytics/utils/utils.ts+++ b/server/src/api/analytics/utils/utils.ts@@ -107,6 +107,7 @@ export async function processResults<T>(results: ResultSet<"JSONEachRow">): Promrow[key] !== "" &&row[key] !== true &&row[key] !== false &&+ !Array.isArray(row[key]) &&!isNaN(Number(row[key]))) {row[key] = Number(row[key]) as any;