🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor analytics queries to use 'events' table instead of 'pageviews'
Bill Yang009c04cee198
diff --git a/server/src/api/analytics/getEvents.ts b/server/src/api/analytics/getEvents.tsindex 5eb7357b0..b55665b55 100644--- a/server/src/api/analytics/getEvents.ts+++ b/server/src/api/analytics/getEvents.ts@@ -61,7 +61,7 @@ export async function getEvents(country,device_type,type- FROM pageviews+ FROM eventsWHEREsite_id = {siteId:Int32}AND (type = 'custom_event' OR type = 'pageview')diff --git a/server/src/api/analytics/getFunnel.ts b/server/src/api/analytics/getFunnel.tsindex 2d96e68aa..295a9fb28 100644--- a/server/src/api/analytics/getFunnel.ts+++ b/server/src/api/analytics/getFunnel.ts@@ -94,7 +94,7 @@ export async function getFunnel(pathname,event_name,type- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${timeStatement}diff --git a/server/src/api/analytics/getLiveSessionLocations.ts b/server/src/api/analytics/getLiveSessionLocations.tsindex 4350ce3d3..2da7780a2 100644--- a/server/src/api/analytics/getLiveSessionLocations.ts+++ b/server/src/api/analytics/getLiveSessionLocations.ts@@ -27,7 +27,7 @@ WITH stuff AS (any(lon) AS lon,any(city) AS cityFROM- pageviews+ eventsWHEREsite_id = {site:Int32}AND timestamp > now() - interval '${req.query.time} minute'diff --git a/server/src/api/analytics/getLiveUsercount.ts b/server/src/api/analytics/getLiveUsercount.tsindex 6ebfb7f47..c58e2c7ec 100644--- a/server/src/api/analytics/getLiveUsercount.ts+++ b/server/src/api/analytics/getLiveUsercount.ts@@ -18,7 +18,7 @@ export const getLiveUsercount = async (}const query = await clickhouse.query({- query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM pageviews WHERE timestamp > now() - interval '${minutes} minute' AND site_id = {siteId:Int32}`,+ query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval '${minutes} minute' AND site_id = {siteId:Int32}`,format: "JSONEachRow",query_params: {siteId: Number(site),diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.tsindex 17f552e09..f1aacd941 100644--- a/server/src/api/analytics/getOverview.ts+++ b/server/src/api/analytics/getOverview.ts@@ -57,7 +57,7 @@ const getQuery = ({MIN(timestamp) AS start_time,MAX(timestamp) AS end_time,COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${filterStatement}@@ -77,7 +77,7 @@ const getQuery = ({SELECTCOUNT(*) AS pageviews,COUNT(DISTINCT user_id) AS users- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${filterStatement}diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.tsindex 021ae96de..db27ad9a1 100644--- a/server/src/api/analytics/getOverviewBucketed.ts+++ b/server/src/api/analytics/getOverviewBucketed.ts@@ -110,7 +110,7 @@ FROMMIN(timestamp) AS start_time,MAX(timestamp) AS end_time,COUNT(*) AS pages_in_session- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${filterStatement}@@ -143,7 +143,7 @@ FULL JOIN}(toTimeZone(timestamp, '${timezone}'))) AS time,COUNT(*) AS pageviews,COUNT(DISTINCT user_id) AS users- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${filterStatement}diff --git a/server/src/api/analytics/getRetention.ts b/server/src/api/analytics/getRetention.tsindex 61ee3d006..fcf28053c 100644--- a/server/src/api/analytics/getRetention.ts+++ b/server/src/api/analytics/getRetention.ts@@ -53,7 +53,7 @@ WITH UserFirstPeriod AS (${periodFunction}(min(timestamp)${retentionMode === "week" ? ", 1" : ""}) AS cohort_period- FROM pageviews+ FROM eventsWHERE site_id = {siteId:UInt16}-- Use the configurable time rangeAND timestamp >= addDays(today(), -{timeRange:UInt16})@@ -65,7 +65,7 @@ PeriodActivity AS (${periodFunction}(timestamp${retentionMode === "week" ? ", 1" : ""}) AS activity_period- FROM pageviews+ FROM eventsWHERE site_id = {siteId:UInt16}-- Match the date range filterAND timestamp >= addDays(today(), -{timeRange:UInt16})diff --git a/server/src/api/analytics/getSession.ts b/server/src/api/analytics/getSession.tsindex 6909d4f76..b4c799a81 100644--- a/server/src/api/analytics/getSession.ts+++ b/server/src/api/analytics/getSession.ts@@ -106,7 +106,7 @@ LIMIT 1const countQuery = `SELECTCOUNT(*) as total-FROM pageviews+FROM eventsWHEREsite_id = ${site}AND session_id = '${sessionId}'@@ -124,7 +124,7 @@ SELECTtype,event_name,properties-FROM pageviews+FROM eventsWHEREsite_id = ${site}AND session_id = '${sessionId}'diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.tsindex a1818aa5a..c1acd3a14 100644--- a/server/src/api/analytics/getSessions.ts+++ b/server/src/api/analytics/getSessions.ts@@ -81,7 +81,7 @@ export async function getSessions(argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,countIf(type = 'pageview') AS pageviews,countIf(type = 'custom_event') AS events- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${userId ? ` AND user_id = '${userId}'` : ""}diff --git a/server/src/api/analytics/getSingleCol.ts b/server/src/api/analytics/getSingleCol.tsindex 477d15bd5..7a509988c 100644--- a/server/src/api/analytics/getSingleCol.ts+++ b/server/src/api/analytics/getSingleCol.ts@@ -50,7 +50,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {event_name as value,COUNT(*) as count,${percentageStatement}- FROM pageviews+ FROM eventsWHEREsite_id = ${site}AND event_name IS NOT NULL@@ -82,7 +82,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {session_id,${arg}(hostname, timestamp) AS hostname,${arg}(pathname, timestamp) AS pathname- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${filterStatement}@@ -106,7 +106,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {${geSqlParam(parameter)} as value,COUNT(distinct(session_id)) as count,${percentageStatement}- FROM pageviews+ FROM eventsWHEREsite_id = ${site}AND ${geSqlParam(parameter)} IS NOT NULLdiff --git a/server/src/api/analytics/getUserInfo.ts b/server/src/api/analytics/getUserInfo.tsindex ee9efe91b..76ab07869 100644--- a/server/src/api/analytics/getUserInfo.ts+++ b/server/src/api/analytics/getUserInfo.ts@@ -58,7 +58,7 @@ export async function getUserInfo(countIf(type = 'pageview') AS pageviews,countIf(type = 'custom_event') AS eventsFROM- pageviews+ eventsWHEREuser_id = {userId:String}AND site_id = {site:Int32}diff --git a/server/src/api/analytics/getUserSessions.ts b/server/src/api/analytics/getUserSessions.tsindex 6eddf8762..22f92ee79 100644--- a/server/src/api/analytics/getUserSessions.ts+++ b/server/src/api/analytics/getUserSessions.ts@@ -80,7 +80,7 @@ SELECTdevice_type,country,iso_3166_2-FROM pageviews+FROM eventsWHEREsite_id = ${site}AND user_id = '${userId}'diff --git a/server/src/api/analytics/getUsers.ts b/server/src/api/analytics/getUsers.tsindex df471737c..b9df01ecd 100644--- a/server/src/api/analytics/getUsers.ts+++ b/server/src/api/analytics/getUsers.ts@@ -100,7 +100,7 @@ WITH AggregatedUsers AS (count(distinct session_id) AS sessions,max(timestamp) AS last_seen,min(timestamp) AS first_seen- FROM pageviews+ FROM eventsWHEREsite_id = ${site}${timeStatement}@@ -118,7 +118,7 @@ LIMIT ${pageSizeNum} OFFSET ${offset}const countQuery = `SELECTcount(DISTINCT user_id) AS total_count-FROM pageviews+FROM eventsWHEREsite_id = ${site}${filterStatement}diff --git a/server/src/api/analytics/utils.ts b/server/src/api/analytics/utils.tsindex 0dc9ce3bd..a605fa684 100644--- a/server/src/api/analytics/utils.ts+++ b/server/src/api/analytics/utils.ts@@ -71,10 +71,10 @@ export const geSqlParam = (parameter: FilterParameter) => {return "domainWithoutWWW(referrer)";}if (parameter === "entry_page") {- return "(SELECT argMin(pathname, timestamp) FROM pageviews WHERE session_id = pageviews.session_id)";+ return "(SELECT argMin(pathname, timestamp) FROM events WHERE session_id = events.session_id)";}if (parameter === "exit_page") {- return "(SELECT argMax(pathname, timestamp) FROM pageviews WHERE session_id = pageviews.session_id)";+ return "(SELECT argMax(pathname, timestamp) FROM events WHERE session_id = events.session_id)";}if (parameter === "dimensions") {return "concat(toString(screen_width), 'x', toString(screen_height))";@@ -107,7 +107,7 @@ export function getFilterStatement(filters: string) {SELECTsession_id,argMin(pathname, timestamp) AS entry_pathname- FROM pageviews+ FROM eventsGROUP BY session_id)WHERE entry_pathname ${filterTypeToOperator(filter.type)} '${x}${@@ -129,7 +129,7 @@ export function getFilterStatement(filters: string) {SELECTsession_id,argMin(pathname, timestamp) AS entry_pathname- FROM pageviews+ FROM eventsGROUP BY session_id)WHERE (${valuesWithOperator.join(" OR ")})@@ -144,7 +144,7 @@ export function getFilterStatement(filters: string) {SELECTsession_id,argMax(pathname, timestamp) AS exit_pathname- FROM pageviews+ FROM eventsGROUP BY session_id)WHERE exit_pathname ${filterTypeToOperator(filter.type)} '${x}${@@ -166,7 +166,7 @@ export function getFilterStatement(filters: string) {SELECTsession_id,argMax(pathname, timestamp) AS exit_pathname- FROM pageviews+ FROM eventsGROUP BY session_id)WHERE (${valuesWithOperator.join(" OR ")})diff --git a/server/src/api/sites/deleteSite.ts b/server/src/api/sites/deleteSite.tsindex 897423eca..e27a92064 100644--- a/server/src/api/sites/deleteSite.ts+++ b/server/src/api/sites/deleteSite.ts@@ -5,6 +5,7 @@ import { sites } from "../../db/postgres/schema.js";import { loadAllowedDomains } from "../../lib/allowedDomains.js";import { getUserHasAccessToSite } from "../../lib/auth-utils.js";import { publicSites } from "../../lib/publicSites.js";+import { clickhouse } from "../../db/clickhouse/clickhouse.js";export async function deleteSite(request: FastifyRequest<{ Params: { id: string } }>,@@ -18,9 +19,9 @@ export async function deleteSite(}await db.delete(sites).where(eq(sites.siteId, Number(id)));- // await clickhouse.query({- // query: `DELETE FROM pageviews WHERE site_id = ${id}`,- // });+ await clickhouse.command({+ query: `DELETE FROM events WHERE site_id = ${id}`,+ });await loadAllowedDomains();// Remove the site from the publicSites cachediff --git a/server/src/api/sites/getSiteHasData.ts b/server/src/api/sites/getSiteHasData.tsindex 07deacb77..452ffd8ab 100644--- a/server/src/api/sites/getSiteHasData.ts+++ b/server/src/api/sites/getSiteHasData.ts@@ -11,7 +11,7 @@ export async function getSiteHasData(// Check if site has data using original methodconst pageviewsData: { count: number }[] = await clickhouse.query({- query: `SELECT count(*) as count FROM pageviews WHERE site_id = ${site}`,+ query: `SELECT count(*) as count FROM events WHERE site_id = ${site}`,format: "JSONEachRow",}).then((res) => res.json());diff --git a/server/src/cron/monthly-usage-checker.ts b/server/src/cron/monthly-usage-checker.tsindex c1e4173d4..097875ae2 100644--- a/server/src/cron/monthly-usage-checker.ts+++ b/server/src/cron/monthly-usage-checker.ts@@ -134,7 +134,7 @@ async function getMonthlyPageviews(const result = await clickhouse.query({query: `SELECT COUNT(*) as count- FROM pageviews+ FROM eventsWHERE site_id IN (${siteIds.join(",")})AND timestamp >= toDate('${periodStart}')`,diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex 794c98c50..dbc8c3ff5 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -7,10 +7,10 @@ export const clickhouse = createClient({});export const initializeClickhouse = async () => {- // Create pageviews table+ // Create events tableawait clickhouse.exec({query: `- CREATE TABLE IF NOT EXISTS pageviews (+ CREATE TABLE IF NOT EXISTS events (site_id UInt16,timestamp DateTime,session_id String,@@ -151,7 +151,7 @@ export const initializeClickhouse = async () => {// any(device_type) AS device_type,// -- Use the largest timestamp as the 'version'// max(toUInt64(timestamp)) AS version- // FROM pageviews+ // FROM events// GROUP BY// site_id,// session_id;diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.tsindex 35bbda799..0d3cc57c4 100644--- a/server/src/tracker/pageviewQueue.ts+++ b/server/src/tracker/pageviewQueue.ts@@ -116,7 +116,7 @@ class PageviewQueue {// Bulk insert into databasetry {await clickhouse.insert({- table: "pageviews",+ table: "events",values: processedPageviews,format: "JSONEachRow",});