Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor analytics queries to use 'events' table instead of 'pageviews'

Bill Yang009c04cee198
diff --git a/server/src/api/analytics/getEvents.ts b/server/src/api/analytics/getEvents.ts
index 5eb7357b0..b55665b55 100644
--- a/server/src/api/analytics/getEvents.ts
+++ b/server/src/api/analytics/getEvents.ts
@@ -61,7 +61,7 @@ export async function getEvents(
country,
device_type,
type
- FROM pageviews
+ FROM events
WHERE
site_id = {siteId:Int32}
AND (type = 'custom_event' OR type = 'pageview')
diff --git a/server/src/api/analytics/getFunnel.ts b/server/src/api/analytics/getFunnel.ts
index 2d96e68aa..295a9fb28 100644
--- a/server/src/api/analytics/getFunnel.ts
+++ b/server/src/api/analytics/getFunnel.ts
@@ -94,7 +94,7 @@ export async function getFunnel(
pathname,
event_name,
type
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${timeStatement}
diff --git a/server/src/api/analytics/getLiveSessionLocations.ts b/server/src/api/analytics/getLiveSessionLocations.ts
index 4350ce3d3..2da7780a2 100644
--- a/server/src/api/analytics/getLiveSessionLocations.ts
+++ b/server/src/api/analytics/getLiveSessionLocations.ts
@@ -27,7 +27,7 @@ WITH stuff AS (
any(lon) AS lon,
any(city) AS city
FROM
- pageviews
+ events
WHERE
site_id = {site:Int32}
AND timestamp > now() - interval '${req.query.time} minute'
diff --git a/server/src/api/analytics/getLiveUsercount.ts b/server/src/api/analytics/getLiveUsercount.ts
index 6ebfb7f47..c58e2c7ec 100644
--- a/server/src/api/analytics/getLiveUsercount.ts
+++ b/server/src/api/analytics/getLiveUsercount.ts
@@ -18,7 +18,7 @@ export const getLiveUsercount = async (
}
const query = await clickhouse.query({
- query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM pageviews WHERE timestamp > now() - interval '${minutes} minute' AND site_id = {siteId:Int32}`,
+ query: `SELECT COUNT(DISTINCT(session_id)) AS count FROM events WHERE timestamp > now() - interval '${minutes} minute' AND site_id = {siteId:Int32}`,
format: "JSONEachRow",
query_params: {
siteId: Number(site),
diff --git a/server/src/api/analytics/getOverview.ts b/server/src/api/analytics/getOverview.ts
index 17f552e09..f1aacd941 100644
--- a/server/src/api/analytics/getOverview.ts
+++ b/server/src/api/analytics/getOverview.ts
@@ -57,7 +57,7 @@ const getQuery = ({
MIN(timestamp) AS start_time,
MAX(timestamp) AS end_time,
COUNT(CASE WHEN type = 'pageview' THEN 1 END) AS pages_in_session
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${filterStatement}
@@ -77,7 +77,7 @@ const getQuery = ({
SELECT
COUNT(*) AS pageviews,
COUNT(DISTINCT user_id) AS users
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${filterStatement}
diff --git a/server/src/api/analytics/getOverviewBucketed.ts b/server/src/api/analytics/getOverviewBucketed.ts
index 021ae96de..db27ad9a1 100644
--- a/server/src/api/analytics/getOverviewBucketed.ts
+++ b/server/src/api/analytics/getOverviewBucketed.ts
@@ -110,7 +110,7 @@ FROM
MIN(timestamp) AS start_time,
MAX(timestamp) AS end_time,
COUNT(*) AS pages_in_session
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${filterStatement}
@@ -143,7 +143,7 @@ FULL JOIN
}(toTimeZone(timestamp, '${timezone}'))) AS time,
COUNT(*) AS pageviews,
COUNT(DISTINCT user_id) AS users
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${filterStatement}
diff --git a/server/src/api/analytics/getRetention.ts b/server/src/api/analytics/getRetention.ts
index 61ee3d006..fcf28053c 100644
--- a/server/src/api/analytics/getRetention.ts
+++ b/server/src/api/analytics/getRetention.ts
@@ -53,7 +53,7 @@ WITH UserFirstPeriod AS (
${periodFunction}(min(timestamp)${
retentionMode === "week" ? ", 1" : ""
}) AS cohort_period
- FROM pageviews
+ FROM events
WHERE site_id = {siteId:UInt16}
-- Use the configurable time range
AND timestamp >= addDays(today(), -{timeRange:UInt16})
@@ -65,7 +65,7 @@ PeriodActivity AS (
${periodFunction}(timestamp${
retentionMode === "week" ? ", 1" : ""
}) AS activity_period
- FROM pageviews
+ FROM events
WHERE site_id = {siteId:UInt16}
-- Match the date range filter
AND timestamp >= addDays(today(), -{timeRange:UInt16})
diff --git a/server/src/api/analytics/getSession.ts b/server/src/api/analytics/getSession.ts
index 6909d4f76..b4c799a81 100644
--- a/server/src/api/analytics/getSession.ts
+++ b/server/src/api/analytics/getSession.ts
@@ -106,7 +106,7 @@ LIMIT 1
const countQuery = `
SELECT
COUNT(*) as total
-FROM pageviews
+FROM events
WHERE
site_id = ${site}
AND session_id = '${sessionId}'
@@ -124,7 +124,7 @@ SELECT
type,
event_name,
properties
-FROM pageviews
+FROM events
WHERE
site_id = ${site}
AND session_id = '${sessionId}'
diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.ts
index a1818aa5a..c1acd3a14 100644
--- a/server/src/api/analytics/getSessions.ts
+++ b/server/src/api/analytics/getSessions.ts
@@ -81,7 +81,7 @@ export async function getSessions(
argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,
countIf(type = 'pageview') AS pageviews,
countIf(type = 'custom_event') AS events
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${userId ? ` AND user_id = '${userId}'` : ""}
diff --git a/server/src/api/analytics/getSingleCol.ts b/server/src/api/analytics/getSingleCol.ts
index 477d15bd5..7a509988c 100644
--- a/server/src/api/analytics/getSingleCol.ts
+++ b/server/src/api/analytics/getSingleCol.ts
@@ -50,7 +50,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
event_name as value,
COUNT(*) as count,
${percentageStatement}
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
AND event_name IS NOT NULL
@@ -82,7 +82,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
session_id,
${arg}(hostname, timestamp) AS hostname,
${arg}(pathname, timestamp) AS pathname
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${filterStatement}
@@ -106,7 +106,7 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
${geSqlParam(parameter)} as value,
COUNT(distinct(session_id)) as count,
${percentageStatement}
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
AND ${geSqlParam(parameter)} IS NOT NULL
diff --git a/server/src/api/analytics/getUserInfo.ts b/server/src/api/analytics/getUserInfo.ts
index ee9efe91b..76ab07869 100644
--- a/server/src/api/analytics/getUserInfo.ts
+++ b/server/src/api/analytics/getUserInfo.ts
@@ -58,7 +58,7 @@ export async function getUserInfo(
countIf(type = 'pageview') AS pageviews,
countIf(type = 'custom_event') AS events
FROM
- pageviews
+ events
WHERE
user_id = {userId:String}
AND site_id = {site:Int32}
diff --git a/server/src/api/analytics/getUserSessions.ts b/server/src/api/analytics/getUserSessions.ts
index 6eddf8762..22f92ee79 100644
--- a/server/src/api/analytics/getUserSessions.ts
+++ b/server/src/api/analytics/getUserSessions.ts
@@ -80,7 +80,7 @@ SELECT
device_type,
country,
iso_3166_2
-FROM pageviews
+FROM events
WHERE
site_id = ${site}
AND user_id = '${userId}'
diff --git a/server/src/api/analytics/getUsers.ts b/server/src/api/analytics/getUsers.ts
index df471737c..b9df01ecd 100644
--- a/server/src/api/analytics/getUsers.ts
+++ b/server/src/api/analytics/getUsers.ts
@@ -100,7 +100,7 @@ WITH AggregatedUsers AS (
count(distinct session_id) AS sessions,
max(timestamp) AS last_seen,
min(timestamp) AS first_seen
- FROM pageviews
+ FROM events
WHERE
site_id = ${site}
${timeStatement}
@@ -118,7 +118,7 @@ LIMIT ${pageSizeNum} OFFSET ${offset}
const countQuery = `
SELECT
count(DISTINCT user_id) AS total_count
-FROM pageviews
+FROM events
WHERE
site_id = ${site}
${filterStatement}
diff --git a/server/src/api/analytics/utils.ts b/server/src/api/analytics/utils.ts
index 0dc9ce3bd..a605fa684 100644
--- a/server/src/api/analytics/utils.ts
+++ b/server/src/api/analytics/utils.ts
@@ -71,10 +71,10 @@ export const geSqlParam = (parameter: FilterParameter) => {
return "domainWithoutWWW(referrer)";
}
if (parameter === "entry_page") {
- return "(SELECT argMin(pathname, timestamp) FROM pageviews WHERE session_id = pageviews.session_id)";
+ return "(SELECT argMin(pathname, timestamp) FROM events WHERE session_id = events.session_id)";
}
if (parameter === "exit_page") {
- return "(SELECT argMax(pathname, timestamp) FROM pageviews WHERE session_id = pageviews.session_id)";
+ return "(SELECT argMax(pathname, timestamp) FROM events WHERE session_id = events.session_id)";
}
if (parameter === "dimensions") {
return "concat(toString(screen_width), 'x', toString(screen_height))";
@@ -107,7 +107,7 @@ export function getFilterStatement(filters: string) {
SELECT
session_id,
argMin(pathname, timestamp) AS entry_pathname
- FROM pageviews
+ FROM events
GROUP BY session_id
)
WHERE entry_pathname ${filterTypeToOperator(filter.type)} '${x}${
@@ -129,7 +129,7 @@ export function getFilterStatement(filters: string) {
SELECT
session_id,
argMin(pathname, timestamp) AS entry_pathname
- FROM pageviews
+ FROM events
GROUP BY session_id
)
WHERE (${valuesWithOperator.join(" OR ")})
@@ -144,7 +144,7 @@ export function getFilterStatement(filters: string) {
SELECT
session_id,
argMax(pathname, timestamp) AS exit_pathname
- FROM pageviews
+ FROM events
GROUP BY session_id
)
WHERE exit_pathname ${filterTypeToOperator(filter.type)} '${x}${
@@ -166,7 +166,7 @@ export function getFilterStatement(filters: string) {
SELECT
session_id,
argMax(pathname, timestamp) AS exit_pathname
- FROM pageviews
+ FROM events
GROUP BY session_id
)
WHERE (${valuesWithOperator.join(" OR ")})
diff --git a/server/src/api/sites/deleteSite.ts b/server/src/api/sites/deleteSite.ts
index 897423eca..e27a92064 100644
--- a/server/src/api/sites/deleteSite.ts
+++ b/server/src/api/sites/deleteSite.ts
@@ -5,6 +5,7 @@ import { sites } from "../../db/postgres/schema.js";
import { loadAllowedDomains } from "../../lib/allowedDomains.js";
import { getUserHasAccessToSite } from "../../lib/auth-utils.js";
import { publicSites } from "../../lib/publicSites.js";
+import { clickhouse } from "../../db/clickhouse/clickhouse.js";
export async function deleteSite(
request: FastifyRequest<{ Params: { id: string } }>,
@@ -18,9 +19,9 @@ export async function deleteSite(
}
await db.delete(sites).where(eq(sites.siteId, Number(id)));
- // await clickhouse.query({
- // query: `DELETE FROM pageviews WHERE site_id = ${id}`,
- // });
+ await clickhouse.command({
+ query: `DELETE FROM events WHERE site_id = ${id}`,
+ });
await loadAllowedDomains();
// Remove the site from the publicSites cache
diff --git a/server/src/api/sites/getSiteHasData.ts b/server/src/api/sites/getSiteHasData.ts
index 07deacb77..452ffd8ab 100644
--- a/server/src/api/sites/getSiteHasData.ts
+++ b/server/src/api/sites/getSiteHasData.ts
@@ -11,7 +11,7 @@ export async function getSiteHasData(
// Check if site has data using original method
const pageviewsData: { count: number }[] = await clickhouse
.query({
- query: `SELECT count(*) as count FROM pageviews WHERE site_id = ${site}`,
+ query: `SELECT count(*) as count FROM events WHERE site_id = ${site}`,
format: "JSONEachRow",
})
.then((res) => res.json());
diff --git a/server/src/cron/monthly-usage-checker.ts b/server/src/cron/monthly-usage-checker.ts
index c1e4173d4..097875ae2 100644
--- a/server/src/cron/monthly-usage-checker.ts
+++ b/server/src/cron/monthly-usage-checker.ts
@@ -134,7 +134,7 @@ async function getMonthlyPageviews(
const result = await clickhouse.query({
query: `
SELECT COUNT(*) as count
- FROM pageviews
+ FROM events
WHERE site_id IN (${siteIds.join(",")})
AND timestamp >= toDate('${periodStart}')
`,
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index 794c98c50..dbc8c3ff5 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -7,10 +7,10 @@ export const clickhouse = createClient({
});
export const initializeClickhouse = async () => {
- // Create pageviews table
+ // Create events table
await clickhouse.exec({
query: `
- CREATE TABLE IF NOT EXISTS pageviews (
+ CREATE TABLE IF NOT EXISTS events (
site_id UInt16,
timestamp DateTime,
session_id String,
@@ -151,7 +151,7 @@ export const initializeClickhouse = async () => {
// any(device_type) AS device_type,
// -- Use the largest timestamp as the 'version'
// max(toUInt64(timestamp)) AS version
- // FROM pageviews
+ // FROM events
// GROUP BY
// site_id,
// session_id;
diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.ts
index 35bbda799..0d3cc57c4 100644
--- a/server/src/tracker/pageviewQueue.ts
+++ b/server/src/tracker/pageviewQueue.ts
@@ -116,7 +116,7 @@ class PageviewQueue {
// Bulk insert into database
try {
await clickhouse.insert({
- table: "pageviews",
+ table: "events",
values: processedPageviews,
format: "JSONEachRow",
});