Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Remove prev column; use psid based calculation for user flow

Blue Mouse0342790ef988
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.ts
index 0ee13bdd6..46b461399 100644
--- a/backend/apps/production/src/analytics/analytics.controller.ts
+++ b/backend/apps/production/src/analytics/analytics.controller.ts
@@ -1323,7 +1323,6 @@ export class AnalyticsController {
psid,
logDTO.pid,
logDTO.pg,
- logDTO.prev,
dv,
br,
brv,
@@ -1456,7 +1455,6 @@ export class AnalyticsController {
psid,
logDTO.pid,
null,
- null,
dv,
br,
brv,
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.ts
index 107d4510c..4f29c66c0 100644
--- a/backend/apps/production/src/analytics/analytics.service.ts
+++ b/backend/apps/production/src/analytics/analytics.service.ts
@@ -672,19 +672,30 @@ export class AnalyticsService {
filtersQuery: string,
): Promise<IUserFlow> {
const query = `
+ WITH page_sequences AS (
+ SELECT
+ psid,
+ pg,
+ created,
+ lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND pg IS NOT NULL
+ ${filtersQuery}
+ )
SELECT
- pg AS source,
- prev AS target,
+ prev_page AS source,
+ pg AS target,
count() AS value
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND pg != prev
- ${filtersQuery}
+ FROM page_sequences
+ WHERE prev_page IS NOT NULL
+ AND prev_page != pg
GROUP BY
- pg,
- prev
+ source,
+ target
+ ORDER BY value DESC
`
const { data } = await clickhouse
diff --git a/backend/apps/production/src/analytics/dto/pageviews.dto.ts b/backend/apps/production/src/analytics/dto/pageviews.dto.ts
index b08793ac2..a9fcb090e 100644
--- a/backend/apps/production/src/analytics/dto/pageviews.dto.ts
+++ b/backend/apps/production/src/analytics/dto/pageviews.dto.ts
@@ -39,12 +39,6 @@ export class PageviewsDto {
})
pg?: string
- @ApiProperty({
- example: '/home',
- description: 'Previous page user was on',
- })
- prev?: string
-
@ApiProperty({
example: 'en-GB',
description: "User's locale",
diff --git a/backend/apps/production/src/analytics/utils/transformers.ts b/backend/apps/production/src/analytics/utils/transformers.ts
index 6a1bb27e0..9056d3740 100644
--- a/backend/apps/production/src/analytics/utils/transformers.ts
+++ b/backend/apps/production/src/analytics/utils/transformers.ts
@@ -27,7 +27,6 @@ export const trafficTransformer = (
psid: string,
pid: string,
pg: string | null,
- prev: string | null,
dv: string | null,
br: string | null,
brv: string | null,
@@ -50,7 +49,6 @@ export const trafficTransformer = (
psid,
pid,
pg: pg || null,
- prev: prev || null,
dv: dv || null,
br: br || null,
brv: brv || null,
diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
index f945fce43..023207bcd 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -1041,7 +1041,6 @@ export class AnalyticsController {
psid,
logDTO.pid,
logDTO.pg,
- logDTO.prev,
dv,
br,
brv,
@@ -1174,7 +1173,6 @@ export class AnalyticsController {
psid,
logDTO.pid,
null,
- null,
dv,
br,
brv,
diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.ts
index 18278aba2..5c225ed10 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts
@@ -632,19 +632,30 @@ export class AnalyticsService {
filtersQuery: string,
): Promise<IUserFlow> {
const query = `
+ WITH page_sequences AS (
+ SELECT
+ psid,
+ pg,
+ created,
+ lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND pg IS NOT NULL
+ ${filtersQuery}
+ )
SELECT
- pg AS source,
- prev AS target,
+ prev_page AS source,
+ pg AS target,
count() AS value
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND pg != prev
- ${filtersQuery}
+ FROM page_sequences
+ WHERE prev_page IS NOT NULL
+ AND prev_page != pg
GROUP BY
- pg,
- prev
+ source,
+ target
+ ORDER BY value DESC
`
const { data } = await clickhouse
@@ -652,7 +663,7 @@ export class AnalyticsService {
query,
query_params: params,
})
- .then(resultSet => resultSet.json<IUserFlowLink>())
+ .then(res => res.json<IUserFlowLink>())
if (_isEmpty(data)) {
const empty = { nodes: [], links: [] }
diff --git a/backend/apps/selfhosted/src/analytics/dto/pageviews.dto.ts b/backend/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
index b08793ac2..a9fcb090e 100644
--- a/backend/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
+++ b/backend/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
@@ -39,12 +39,6 @@ export class PageviewsDto {
})
pg?: string
- @ApiProperty({
- example: '/home',
- description: 'Previous page user was on',
- })
- prev?: string
-
@ApiProperty({
example: 'en-GB',
description: "User's locale",
diff --git a/backend/apps/selfhosted/src/analytics/utils/transformers.ts b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
index 6a1bb27e0..9056d3740 100644
--- a/backend/apps/selfhosted/src/analytics/utils/transformers.ts
+++ b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
@@ -27,7 +27,6 @@ export const trafficTransformer = (
psid: string,
pid: string,
pg: string | null,
- prev: string | null,
dv: string | null,
br: string | null,
brv: string | null,
@@ -50,7 +49,6 @@ export const trafficTransformer = (
psid,
pid,
pg: pg || null,
- prev: prev || null,
dv: dv || null,
br: br || null,
brv: brv || null,
diff --git a/backend/migrations/clickhouse/2024_10_26.js b/backend/migrations/clickhouse/2024_10_26.js
new file mode 100644
index 000000000..733c806b2
--- /dev/null
+++ b/backend/migrations/clickhouse/2024_10_26.js
@@ -0,0 +1,8 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Remove prev column from analytics table
+ `ALTER TABLE ${dbName}.analytics DROP COLUMN IF EXISTS prev;`,
+]
+
+queriesRunner(queries)
diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js
index eac121cf6..670f67f90 100644
--- a/backend/migrations/clickhouse/initialise_database.js
+++ b/backend/migrations/clickhouse/initialise_database.js
@@ -10,7 +10,6 @@ const CLICKHOUSE_INIT_QUERIES = [
psid Nullable(UInt64),
pid FixedString(12),
pg Nullable(String),
- prev Nullable(String),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
brv Nullable(String),