Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(kind of) fix birdseye & log endpoint return different numbers (this fucking code is 4 years old, one day it should all be rewritten from scratch and become readable to humans)

Blue Mouse487e92eac6b5
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts
index 82ae1db27..ab393f49c 100644
--- a/backend/apps/cloud/src/analytics/analytics.controller.ts
+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts
@@ -781,6 +781,7 @@ export class AnalyticsController {
from,
to,
timezone = DEFAULT_TIMEZONE,
+ timeBucket,
filters,
} = data
const pidsArray = getPIDsArray(pids, pid)
@@ -814,6 +815,7 @@ export class AnalyticsController {
return this.analyticsService.getAnalyticsSummary(
validPids,
+ timeBucket,
period,
from,
to,
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 67f363aca..cea5661d0 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -521,6 +521,7 @@ export class AnalyticsService {
period: string,
safeTimezone: string,
diff?: number,
+ checkTimebucket = true,
): IGetGroupFromTo {
let groupFrom: dayjs.Dayjs
let groupTo: dayjs.Dayjs
@@ -551,27 +552,10 @@ export class AnalyticsService {
)
}
- if (!_isEmpty(timeBucket)) {
+ if (!_isEmpty(timeBucket) && checkTimebucket) {
checkIfTBAllowed(timeBucket, from, to)
}
- // TODO:
- // THE FOLLOWING CODE SHOULD BE TIMEZONE SPECIFIC BUT HAS SOME ISSUES WITH DATE SHIFTS
- // IT SHOULD BE REFACTORED AND USED IN THE FUTURE
- // if (from === to) {
- // // When from and to are the same we need to return timezone specific data for 1 specified day
- // groupFrom = dayjs.tz(from, safeTimezone).startOf('d')
- // groupTo = groupFrom.endOf('d')
- // groupFromUTC = groupFrom.utc().format(formatFrom)
- // groupToUTC = groupTo.utc().format(formatTo)
- // } else {
- // groupFrom = dayjs.tz(from, safeTimezone)
- // groupTo = dayjs.tz(to, safeTimezone)
- // groupFromUTC = groupFrom.utc().startOf(timeBucket).format(formatFrom)
- // groupToUTC = groupTo.utc().endOf(timeBucket).format(formatTo)
- // }
-
- // THIS SHOULD BE REPLACED BY THE CODE ABOVE WHEN IT'S FIXED
if (from === to) {
groupFrom = dayjs.tz(from, safeTimezone).startOf('d')
groupTo = dayjs.tz(from, safeTimezone).endOf('d')
@@ -613,7 +597,7 @@ export class AnalyticsService {
groupTo = djsNow
}
- if (!_isEmpty(timeBucket)) {
+ if (!_isEmpty(timeBucket) && checkTimebucket) {
checkIfTBAllowed(
timeBucket,
groupFrom.format(formatFrom),
@@ -1425,30 +1409,26 @@ export class AnalyticsService {
async getAnalyticsSummary(
pids: string[],
+ timeBucket?: string,
period?: string,
from?: string,
to?: string,
timezone?: string,
filters?: string,
): Promise<IOverall> {
- let _from: string
-
- let _to: string
-
- if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {
- const safeTimezone = this.getSafeTimezone(timezone)
-
- const { groupFrom, groupTo } = this.getGroupFromTo(
- from,
- to,
- ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,
- period,
- safeTimezone,
- )
+ const safeTimezone = this.getSafeTimezone(timezone)
- _from = groupFrom
- _to = groupTo
- }
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ ['today', 'yesterday', 'custom'].includes(period)
+ ? TimeBucketType.HOUR
+ : (timeBucket as TimeBucketType) || TimeBucketType.DAY,
+ period,
+ safeTimezone,
+ undefined,
+ false,
+ )
const result = {}
@@ -1528,30 +1508,13 @@ export class AnalyticsService {
return
}
- let now: string
- let periodFormatted: string
- let periodSubtracted: string
-
- if (_from && _to) {
- // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
- const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
-
- now = _to
- periodFormatted = _from
- periodSubtracted = dayjs(_from)
- .subtract(diff, 'days')
- .format('YYYY-MM-DD HH:mm:ss')
- } else {
- const amountToSubtract = parseInt(period, 10)
- const unit = _replace(period, /[0-9]/g, '') as dayjs.ManipulateType
-
- now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
- periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
- periodSubtracted = periodRaw
- .subtract(amountToSubtract, unit)
- .format('YYYY-MM-DD HH:mm:ss')
- }
+ const periodSubtracted = dayjs
+ .utc(groupToUTC)
+ .subtract(
+ dayjs.utc(groupFromUTC).diff(dayjs.utc(groupToUTC), 'minutes'),
+ 'minutes',
+ )
+ .format('YYYY-MM-DD HH:mm:ss')
let queryCurrent = `
WITH analytics_counts AS (
@@ -1562,7 +1525,7 @@ export class AnalyticsService {
FROM analytics
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
),
duration_avg AS (
@@ -1573,7 +1536,7 @@ export class AnalyticsService {
SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
)
)
@@ -1592,7 +1555,7 @@ export class AnalyticsService {
FROM analytics
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
),
duration_avg AS (
@@ -1603,7 +1566,7 @@ export class AnalyticsService {
SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
)
)
@@ -1619,7 +1582,7 @@ export class AnalyticsService {
FROM customEV
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
`
queryPrevious = `
@@ -1627,7 +1590,7 @@ export class AnalyticsService {
FROM customEV
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
`
}
@@ -1639,9 +1602,9 @@ export class AnalyticsService {
query,
query_params: {
pid,
- periodFormatted,
+ groupFromUTC,
+ groupToUTC,
periodSubtracted,
- now,
...filtersParams,
},
})
diff --git a/backend/apps/cloud/src/analytics/dto/get-overall-stats.dto.ts b/backend/apps/cloud/src/analytics/dto/get-overall-stats.dto.ts
index 3b1e39f22..1460d9039 100644
--- a/backend/apps/cloud/src/analytics/dto/get-overall-stats.dto.ts
+++ b/backend/apps/cloud/src/analytics/dto/get-overall-stats.dto.ts
@@ -18,6 +18,14 @@ export class GetOverallStatsDto {
@Matches(PID_REGEX, { message: 'The provided Project ID (pid) is incorrect' })
pid?: string
+ @ApiProperty({
+ description: 'Time bucket to display data in',
+ required: false,
+ })
+ @IsOptional()
+ @IsString()
+ timeBucket?: string
+
@ApiProperty({ required: false })
@IsOptional()
@ValidatePeriod()
diff --git a/backend/apps/cloud/src/integrations/telegram/scene/projects.scene.ts b/backend/apps/cloud/src/integrations/telegram/scene/projects.scene.ts
index 9b1dd056c..0cf8efdb1 100644
--- a/backend/apps/cloud/src/integrations/telegram/scene/projects.scene.ts
+++ b/backend/apps/cloud/src/integrations/telegram/scene/projects.scene.ts
@@ -81,6 +81,7 @@ export class ProjectsScene {
)
const stats = await this.analyticsService.getAnalyticsSummary(
[project.id],
+ undefined,
'7d',
)
diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts
index baaa04eb5..c513b34f5 100644
--- a/backend/apps/cloud/src/task-manager/task-manager.service.ts
+++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts
@@ -343,6 +343,7 @@ export class TaskManagerService {
const data = this.analyticsService.convertSummaryToObsoleteFormat(
await this.analyticsService.getAnalyticsSummary(
ids,
+ undefined,
params.analyticsParam,
),
)
@@ -401,6 +402,7 @@ export class TaskManagerService {
const data = this.analyticsService.convertSummaryToObsoleteFormat(
await this.analyticsService.getAnalyticsSummary(
ids,
+ undefined,
params.analyticsParam,
),
)
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index b620859dd..a3de9627c 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -688,6 +688,7 @@ export class AnalyticsController {
to,
timezone = DEFAULT_TIMEZONE,
filters,
+ timeBucket,
} = data
const pidsArray = getPIDsArray(pids, pid)
@@ -703,6 +704,7 @@ export class AnalyticsController {
return this.analyticsService.getAnalyticsSummary(
pidsArray,
+ timeBucket,
period,
from,
to,
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index 110cdfc2c..5fbed0833 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -500,6 +500,7 @@ export class AnalyticsService {
period: string,
safeTimezone: string,
diff?: number,
+ checkTimebucket = true,
): IGetGroupFromTo {
let groupFrom: dayjs.Dayjs
let groupTo: dayjs.Dayjs
@@ -530,27 +531,10 @@ export class AnalyticsService {
)
}
- if (!_isEmpty(timeBucket)) {
+ if (!_isEmpty(timeBucket) && checkTimebucket) {
checkIfTBAllowed(timeBucket, from, to)
}
- // TODO:
- // THE FOLLOWING CODE SHOULD BE TIMEZONE SPECIFIC BUT HAS SOME ISSUES WITH DATE SHIFTS
- // IT SHOULD BE REFACTORED AND USED IN THE FUTURE
- // if (from === to) {
- // // When from and to are the same we need to return timezone specific data for 1 specified day
- // groupFrom = dayjs.tz(from, safeTimezone).startOf('d')
- // groupTo = groupFrom.endOf('d')
- // groupFromUTC = groupFrom.utc().format(formatFrom)
- // groupToUTC = groupTo.utc().format(formatTo)
- // } else {
- // groupFrom = dayjs.tz(from, safeTimezone)
- // groupTo = dayjs.tz(to, safeTimezone)
- // groupFromUTC = groupFrom.utc().startOf(timeBucket).format(formatFrom)
- // groupToUTC = groupTo.utc().endOf(timeBucket).format(formatTo)
- // }
-
- // THIS SHOULD BE REPLACED BY THE CODE ABOVE WHEN IT'S FIXED
if (from === to) {
groupFrom = dayjs.tz(from, safeTimezone).startOf('d')
groupTo = dayjs.tz(from, safeTimezone).endOf('d')
@@ -592,7 +576,7 @@ export class AnalyticsService {
groupTo = djsNow
}
- if (!_isEmpty(timeBucket)) {
+ if (!_isEmpty(timeBucket) && checkTimebucket) {
checkIfTBAllowed(
timeBucket,
groupFrom.format(formatFrom),
@@ -1256,30 +1240,26 @@ export class AnalyticsService {
async getAnalyticsSummary(
pids: string[],
+ timeBucket?: string,
period?: string,
from?: string,
to?: string,
timezone?: string,
filters?: string,
): Promise<IOverall> {
- let _from: string
-
- let _to: string
-
- if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {
- const safeTimezone = this.getSafeTimezone(timezone)
-
- const { groupFrom, groupTo } = this.getGroupFromTo(
- from,
- to,
- ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,
- period,
- safeTimezone,
- )
+ const safeTimezone = this.getSafeTimezone(timezone)
- _from = groupFrom
- _to = groupTo
- }
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ ['today', 'yesterday', 'custom'].includes(period)
+ ? TimeBucketType.HOUR
+ : (timeBucket as TimeBucketType) || TimeBucketType.DAY,
+ period,
+ safeTimezone,
+ undefined,
+ false,
+ )
const result = {}
@@ -1359,30 +1339,13 @@ export class AnalyticsService {
return
}
- let now: string
- let periodFormatted: string
- let periodSubtracted: string
-
- if (_from && _to) {
- // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
- const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
-
- now = _to
- periodFormatted = _from
- periodSubtracted = dayjs(_from)
- .subtract(diff, 'days')
- .format('YYYY-MM-DD HH:mm:ss')
- } else {
- const amountToSubtract = parseInt(period, 10)
- const unit = _replace(period, /[0-9]/g, '') as dayjs.ManipulateType
-
- now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
- periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
- periodSubtracted = periodRaw
- .subtract(amountToSubtract, unit)
- .format('YYYY-MM-DD HH:mm:ss')
- }
+ const periodSubtracted = dayjs
+ .utc(groupToUTC)
+ .subtract(
+ dayjs.utc(groupFromUTC).diff(dayjs.utc(groupToUTC), 'minutes'),
+ 'minutes',
+ )
+ .format('YYYY-MM-DD HH:mm:ss')
let queryCurrent = `
WITH analytics_counts AS (
@@ -1393,7 +1356,7 @@ export class AnalyticsService {
FROM analytics
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
),
duration_avg AS (
@@ -1404,7 +1367,7 @@ export class AnalyticsService {
SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
)
)
@@ -1423,7 +1386,7 @@ export class AnalyticsService {
FROM analytics
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
),
duration_avg AS (
@@ -1434,7 +1397,7 @@ export class AnalyticsService {
SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
)
)
@@ -1450,7 +1413,7 @@ export class AnalyticsService {
FROM customEV
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
`
queryPrevious = `
@@ -1458,7 +1421,7 @@ export class AnalyticsService {
FROM customEV
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
`
}
@@ -1470,9 +1433,9 @@ export class AnalyticsService {
query,
query_params: {
pid,
- periodFormatted,
+ groupFromUTC,
+ groupToUTC,
periodSubtracted,
- now,
...filtersParams,
},
})
diff --git a/backend/apps/community/src/analytics/dto/get-overall-stats.dto.ts b/backend/apps/community/src/analytics/dto/get-overall-stats.dto.ts
index 3b1e39f22..1460d9039 100644
--- a/backend/apps/community/src/analytics/dto/get-overall-stats.dto.ts
+++ b/backend/apps/community/src/analytics/dto/get-overall-stats.dto.ts
@@ -18,6 +18,14 @@ export class GetOverallStatsDto {
@Matches(PID_REGEX, { message: 'The provided Project ID (pid) is incorrect' })
pid?: string
+ @ApiProperty({
+ description: 'Time bucket to display data in',
+ required: false,
+ })
+ @IsOptional()
+ @IsString()
+ timeBucket?: string
+
@ApiProperty({ required: false })
@IsOptional()
@ValidatePeriod()
diff --git a/web/app/api/index.ts b/web/app/api/index.ts
index ade7fdb20..b9a701624 100644
--- a/web/app/api/index.ts
+++ b/web/app/api/index.ts
@@ -790,6 +790,7 @@ export const getCaptchaData = (pid: string, tb = 'hour', period = '3d', filters:
export const getOverallStats = (
pids: string[],
+ tb: string,
period: string,
from = '',
to = '',
@@ -801,7 +802,7 @@ export const getOverallStats = (
.get(
`log/birdseye?pids=[${_map(pids, (pid) => `"${pid}"`).join(
',',
- )}]&period=${period}&from=${from}&to=${to}&timezone=${timezone}&filters=${JSON.stringify(filters)}`,
+ )}]&timeBucket=${tb}&period=${period}&from=${from}&to=${to}&timezone=${timezone}&filters=${JSON.stringify(filters)}`,
{
headers: {
'x-password': password,
diff --git a/web/app/pages/Dashboard/Dashboard.tsx b/web/app/pages/Dashboard/Dashboard.tsx
index 11c1bd74d..2486d7700 100644
--- a/web/app/pages/Dashboard/Dashboard.tsx
+++ b/web/app/pages/Dashboard/Dashboard.tsx
@@ -17,7 +17,7 @@ import { withAuthentication, auth } from '~/hoc/protected'
import useBreakpoint from '~/hooks/useBreakpoint'
import useDebounce from '~/hooks/useDebounce'
import useFeatureFlag from '~/hooks/useFeatureFlag'
-import { isSelfhosted, LIVE_VISITORS_UPDATE_INTERVAL } from '~/lib/constants'
+import { isSelfhosted, LIVE_VISITORS_UPDATE_INTERVAL, tbPeriodPairs } from '~/lib/constants'
import { Overall, Project } from '~/lib/models/Project'
import { FeatureFlag } from '~/lib/models/User'
import { useAuth } from '~/providers/AuthProvider'
@@ -101,6 +101,8 @@ const Dashboard = () => {
return sortParam && Object.values(SORT_OPTIONS).includes(sortParam as any) ? sortParam : SORT_OPTIONS.ALPHA_ASC
})
+ const periodPairs = tbPeriodPairs(t)
+
const pageAmount = Math.ceil(paginationTotal / pageSize)
// This search represents what's inside the search input
@@ -236,7 +238,8 @@ const Dashboard = () => {
if (!projectIds.length || isHostnameNavigationEnabled) return
try {
- const stats = await getOverallStats(projectIds, activePeriod)
+ const timeBucket = periodPairs.find((p) => p.period === activePeriod)?.tbs[0] || ''
+ const stats = await getOverallStats(projectIds, timeBucket, activePeriod)
setOverallStats((prev) => ({ ...prev, ...stats }))
} catch (reason) {
console.error('Failed to fetch overall stats:', reason)
@@ -267,7 +270,7 @@ const Dashboard = () => {
const interval = setInterval(updateLiveVisitors, LIVE_VISITORS_UPDATE_INTERVAL)
return () => clearInterval(interval)
- }, [projects, activePeriod, isHostnameNavigationEnabled]) // Reset interval when projects change
+ }, [projects, activePeriod, isHostnameNavigationEnabled, periodPairs]) // Reset interval when projects change
if (error && isLoading === false) {
return (
diff --git a/web/app/pages/Project/View/ViewProject.tsx b/web/app/pages/Project/View/ViewProject.tsx
index 35c4f0c25..22cd3209f 100644
--- a/web/app/pages/Project/View/ViewProject.tsx
+++ b/web/app/pages/Project/View/ViewProject.tsx
@@ -1347,6 +1347,7 @@ const ViewProject = () => {
)) || {}
const compareOverall = await getOverallStats(
[id],
+ timeBucket,
'custom',
fromCompare,
toCompare,
@@ -1391,7 +1392,7 @@ const ViewProject = () => {
projectPassword,
)
}
- rawOverall = await getOverallStats([id], period, from, to, timezone, filters, projectPassword)
+ rawOverall = await getOverallStats([id], timeBucket, period, from, to, timezone, filters, projectPassword)
} else {
data = await getProjectData(
id,
@@ -1418,7 +1419,7 @@ const ViewProject = () => {
projectPassword,
)
}
- rawOverall = await getOverallStats([id], period, '', '', timezone, filters, projectPassword)
+ rawOverall = await getOverallStats([id], timeBucket, period, '', '', timezone, filters, projectPassword)
}
customEventsChart = customEventsChart?.chart ? customEventsChart.chart.events : customEventsChartData
@@ -1834,7 +1835,7 @@ const ViewProject = () => {
if (period === 'custom' && dateRange) {
dataPerf = await getPerfData(id, timeBucket, '', filters, from, to, timezone, measure, projectPassword)
- rawOverall = await getOverallStats([id], period, from, to, timezone, filters, projectPassword)
+ rawOverall = await getOverallStats([id], timeBucket, period, from, to, timezone, filters, projectPassword)
} else {
dataPerf = await getPerfData(id, timeBucket, period, filters, '', '', timezone, measure, projectPassword)
rawOverall = await getPerformanceOverallStats([id], period, '', '', timezone, filters, measure, projectPassword)