Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

add better support for timezones

Carl-Gerhard Lindesvärd33e92a0da0f3
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 1f98d79ee..2d698e35c 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -56,8 +56,8 @@ const startServer = async () => {
onError(error: unknown) {
if (error instanceof Error) {
logger.error(error, error.message);
- } else {
- logger.error(error, 'Unknown error trpc error');
+ } else if (error && typeof error === 'object' && 'error' in error) {
+ logger.error(error.error, 'Unknown error trpc error');
}
},
} satisfies FastifyTRPCPluginOptions<AppRouter>['trpcOptions'],
diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx
index dbc335d25..6722fda93 100644
--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx
+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx
@@ -2,6 +2,7 @@
import { useOverviewOptions } from '@/components/overview/useOverviewOptions';
import { TimeWindowPicker } from '@/components/time-window-picker';
+import { endOfDay, formatISO, startOfDay } from 'date-fns';
export function OverviewReportRange() {
const { range, setRange, setStartDate, setEndDate, endDate, startDate } =
@@ -11,8 +12,14 @@ export function OverviewReportRange() {
<TimeWindowPicker
onChange={setRange}
value={range}
- onStartDateChange={(date) => setStartDate(date)}
- onEndDateChange={(date) => setEndDate(date)}
+ onStartDateChange={(date) => {
+ const d = formatISO(startOfDay(new Date(date)));
+ setStartDate(d);
+ }}
+ onEndDateChange={(date) => {
+ const d = formatISO(endOfDay(new Date(date)));
+ setEndDate(d);
+ }}
endDate={endDate}
startDate={startDate}
/>
diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx
deleted file mode 100644
index 9cf2cf500..000000000
--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx
+++ /dev/null
@@ -1,3 +0,0 @@
-import FullPageLoadingState from '@/components/full-page-loading-state';
-
-export default FullPageLoadingState;
diff --git a/apps/dashboard/src/components/report/reportSlice.ts b/apps/dashboard/src/components/report/reportSlice.ts
index 0616a3316..b63211286 100644
--- a/apps/dashboard/src/components/report/reportSlice.ts
+++ b/apps/dashboard/src/components/report/reportSlice.ts
@@ -1,6 +1,12 @@
import { createSlice } from '@reduxjs/toolkit';
import type { PayloadAction } from '@reduxjs/toolkit';
-import { isSameDay, isSameMonth } from 'date-fns';
+import {
+ endOfDay,
+ formatISO,
+ isSameDay,
+ isSameMonth,
+ startOfDay,
+} from 'date-fns';
import {
alphabetIds,
@@ -188,8 +194,8 @@ export const reportSlice = createSlice({
}>
) => {
state.dirty = true;
- state.startDate = action.payload.startDate;
- state.endDate = action.payload.endDate;
+ state.startDate = formatISO(startOfDay(action.payload.startDate));
+ state.endDate = formatISO(endOfDay(action.payload.endDate));
if (isSameDay(state.startDate, state.endDate)) {
state.interval = 'hour';
@@ -203,7 +209,7 @@ export const reportSlice = createSlice({
// Date range
changeStartDate: (state, action: PayloadAction<string>) => {
state.dirty = true;
- state.startDate = action.payload;
+ state.startDate = formatISO(startOfDay(action.payload));
const interval = getDefaultIntervalByDates(
state.startDate,
@@ -217,7 +223,7 @@ export const reportSlice = createSlice({
// Date range
changeEndDate: (state, action: PayloadAction<string>) => {
state.dirty = true;
- state.endDate = action.payload;
+ state.endDate = formatISO(endOfDay(action.payload));
const interval = getDefaultIntervalByDates(
state.startDate,
diff --git a/packages/common/src/date.ts b/packages/common/src/date.ts
index b2bbdf8dd..2f520cc29 100644
--- a/packages/common/src/date.ts
+++ b/packages/common/src/date.ts
@@ -5,3 +5,41 @@ export function getTime(date: string | number | Date) {
export function toISOString(date: string | number | Date) {
return new Date(date).toISOString();
}
+
+export function getTimezoneFromDateString(_date: string) {
+ const mapper: Record<string, string> = {
+ '+00:00': 'UTC',
+ '+01:00': 'Europe/Paris',
+ '+02:00': 'Europe/Stockholm',
+ '+03:00': 'Europe/Moscow',
+ '+04:00': 'Asia/Dubai',
+ '+05:00': 'Asia/Karachi',
+ '+06:00': 'Asia/Dhaka',
+ '+07:00': 'Asia/Bangkok',
+ '+08:00': 'Asia/Shanghai',
+ '+09:00': 'Asia/Tokyo',
+ '+10:00': 'Australia/Sydney',
+ '+11:00': 'Pacific/Noumea',
+ '+12:00': 'Pacific/Fiji',
+ '-02:00': 'America/Noronha',
+ '-03:00': 'America/Sao_Paulo',
+ '-04:00': 'America/Santiago',
+ '-05:00': 'America/Bogota',
+ '-06:00': 'America/Mexico_City',
+ '-07:00': 'America/Phoenix',
+ '-08:00': 'America/Los_Angeles',
+ '-09:00': 'America/Anchorage',
+ '-10:00': 'Pacific/Honolulu',
+ '-11:00': 'Pacific/Midway',
+ '-12:00': 'Pacific/Tarawa',
+ };
+
+ const defaultTimezone = 'UTC';
+
+ const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0];
+ if (match) {
+ return mapper[match] ?? defaultTimezone;
+ }
+
+ return defaultTimezone;
+}
diff --git a/packages/constants/index.ts b/packages/constants/index.ts
index c2bf653fe..acd51f948 100644
--- a/packages/constants/index.ts
+++ b/packages/constants/index.ts
@@ -15,7 +15,7 @@ export const timeWindows = {
},
today: {
key: 'today',
- label: 'Today',
+ label: '24 hours',
shortcut: 'D',
},
'7d': {
diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.ts
index 2d3064cd6..3966c7551 100644
--- a/packages/db/src/clickhouse-client.ts
+++ b/packages/db/src/clickhouse-client.ts
@@ -15,6 +15,7 @@ export const ch = createClient({
export async function chQueryWithMeta<T extends Record<string, any>>(
query: string
): Promise<ResponseJSON<T>> {
+ console.log('Query:', query);
const start = Date.now();
const res = await ch.query({
query,
@@ -39,7 +40,6 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
console.log(`Clickhouse query took ${response.statistics?.elapsed}ms`);
console.log(`chQuery took ${Date.now() - start}ms`);
- console.log('Query:', query);
return response;
}
@@ -52,7 +52,11 @@ export async function chQuery<T extends Record<string, any>>(
export function formatClickhouseDate(_date: Date | string) {
const date = typeof _date === 'string' ? new Date(_date) : _date;
- return date.toISOString().replace('T', ' ').replace(/Z+$/, '');
+ return date
+ .toISOString()
+ .replace('T', ' ')
+ .replace(/Z+$/, '')
+ .replace(/\.[0-9]+$/, '');
}
export function convertClickhouseDateToJs(date: string) {
diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts
index a49dd787e..be56c9339 100644
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -1,5 +1,6 @@
import { escape } from 'sqlstring';
+import { getTimezoneFromDateString } from '@openpanel/common';
import type {
IChartEventFilter,
IGetChartDataInput,
@@ -21,32 +22,40 @@ export function getChartSql({
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
+
+ let labelValue = escape('*');
if (event.name !== '*') {
- sb.select.label = `${escape(event.name)} as label`;
- sb.where.eventName = `name = ${escape(event.name)}`;
+ labelValue = `${escape(event.name)}`;
+ sb.select.label = `${labelValue} as label`;
+ sb.where.eventName = `name = ${labelValue}`;
+ } else {
+ sb.select.label = `${labelValue} as label`;
}
sb.select.count = `count(*) as count`;
switch (interval) {
case 'minute': {
- sb.select.date = `toStartOfMinute(created_at) as date`;
+ sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'hour': {
- sb.select.date = `toStartOfHour(created_at) as date`;
+ sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'day': {
- sb.select.date = `toStartOfDay(created_at) as date`;
+ sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'month': {
- sb.select.date = `toStartOfMonth(created_at) as date`;
+ sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
}
sb.groupBy.date = 'date';
- sb.orderBy.date = 'date ASC';
if (startDate) {
sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;
diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts
index f2f98c610..44a4d9f3a 100644
--- a/packages/trpc/src/routers/chart.helpers.ts
+++ b/packages/trpc/src/routers/chart.helpers.ts
@@ -1,10 +1,14 @@
import {
+ differenceInMilliseconds,
endOfDay,
+ endOfMonth,
endOfYear,
+ formatISO,
startOfDay,
startOfMonth,
startOfYear,
subDays,
+ subMilliseconds,
subMinutes,
subMonths,
subYears,
@@ -43,129 +47,6 @@ function getEventLegend(event: IChartEvent) {
return event.displayName ?? `${event.name} (${event.id})`;
}
-function fillEmptySpotsInTimeline(
- items: ResultItem[],
- interval: IInterval,
- startDate: string,
- endDate: string
-) {
- const result = [];
- const clonedStartDate = new Date(startDate);
- const clonedEndDate = new Date(endDate);
- const today = new Date();
-
- if (interval === 'minute') {
- clonedStartDate.setUTCSeconds(0, 0);
- clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0);
- } else if (interval === 'hour') {
- clonedStartDate.setUTCMinutes(0, 0, 0);
- clonedEndDate.setUTCMinutes(0, 0, 0);
- } else {
- clonedStartDate.setUTCHours(0, 0, 0, 0);
- clonedEndDate.setUTCHours(0, 0, 0, 0);
- }
-
- if (interval === 'month') {
- clonedStartDate.setUTCDate(1);
- clonedEndDate.setUTCDate(1);
- }
-
- // Force if interval is month and the start date is the same month as today
- const shouldForce = () =>
- interval === 'month' &&
- clonedStartDate.getUTCFullYear() === today.getUTCFullYear() &&
- clonedStartDate.getUTCMonth() === today.getUTCMonth();
- let prev = undefined;
- while (
- shouldForce() ||
- clonedStartDate.getTime() <= clonedEndDate.getTime()
- ) {
- if (prev === clonedStartDate.getTime()) {
- break;
- }
- prev = clonedStartDate.getTime();
-
- const getYear = (date: Date) => date.getUTCFullYear();
- const getMonth = (date: Date) => date.getUTCMonth();
- const getDay = (date: Date) => date.getUTCDate();
- const getHour = (date: Date) => date.getUTCHours();
- const getMinute = (date: Date) => date.getUTCMinutes();
-
- const item = items.find((item) => {
- const date = convertClickhouseDateToJs(item.date);
-
- if (interval === 'month') {
- return (
- getYear(date) === getYear(clonedStartDate) &&
- getMonth(date) === getMonth(clonedStartDate)
- );
- }
- if (interval === 'day') {
- return (
- getYear(date) === getYear(clonedStartDate) &&
- getMonth(date) === getMonth(clonedStartDate) &&
- getDay(date) === getDay(clonedStartDate)
- );
- }
- if (interval === 'hour') {
- return (
- getYear(date) === getYear(clonedStartDate) &&
- getMonth(date) === getMonth(clonedStartDate) &&
- getDay(date) === getDay(clonedStartDate) &&
- getHour(date) === getHour(clonedStartDate)
- );
- }
- if (interval === 'minute') {
- return (
- getYear(date) === getYear(clonedStartDate) &&
- getMonth(date) === getMonth(clonedStartDate) &&
- getDay(date) === getDay(clonedStartDate) &&
- getHour(date) === getHour(clonedStartDate) &&
- getMinute(date) === getMinute(clonedStartDate)
- );
- }
-
- return false;
- });
-
- if (item) {
- result.push({
- ...item,
- date: clonedStartDate.toISOString(),
- });
- } else {
- result.push({
- date: clonedStartDate.toISOString(),
- count: 0,
- label: null,
- });
- }
-
- switch (interval) {
- case 'day': {
- clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1);
- break;
- }
- case 'hour': {
- clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1);
- break;
- }
- case 'minute': {
- clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1);
- break;
- }
- case 'month': {
- clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1);
- break;
- }
- }
- }
-
- return sort(function (a, b) {
- return new Date(a.date).getTime() - new Date(b.date).getTime();
- }, result);
-}
-
export function withFormula(
{ formula, events }: IChartInput,
series: GetChartDataResult
@@ -235,6 +116,26 @@ export function withFormula(
];
}
+const toDynamicISODateWithTZ = (
+ date: string,
+ blueprint: string,
+ interval: IInterval
+) => {
+ // If we have a space in the date we know it's a date with time
+ if (date.includes(' ')) {
+ // If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server)
+ // - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.)
+ // - We use timezone from client if its a custom range
+ if (interval === 'minute' || interval === 'hour') {
+ return date.replace(' ', 'T') + blueprint.slice(-6);
+ }
+ // Otherwise we just return without the timezone
+ // It will be converted to the correct timezone on the client
+ return date.replace(' ', 'T');
+ }
+ return `${date}T00:00:00`;
+};
+
export async function getChartData(payload: IGetChartDataInput) {
let result = await chQuery<ResultItem>(getChartSql(payload));
@@ -250,9 +151,15 @@ export async function getChartData(payload: IGetChartDataInput) {
// group by sql label
const series = result.reduce(
(acc, item) => {
+ // If we fill empty spots in the timeline (clickhouse) we wont get a label back
+ // take the event name as label
+ if (!item.label && item.count === 0) {
+ item.label = payload.event.name;
+ }
+
+ const label = item.label?.trim() || NOT_SET_VALUE;
// item.label can be null when using breakdowns on a property
// that doesn't exist on all events
- const label = item.label?.trim() || NOT_SET_VALUE;
if (label) {
if (acc[label]) {
acc[label]?.push(item);
@@ -281,22 +188,25 @@ export async function getChartData(payload: IGetChartDataInput) {
payload.chartType === 'metric' ||
payload.chartType === 'pie' ||
payload.chartType === 'bar'
- ? fillEmptySpotsInTimeline(
- series[key] ?? [],
- payload.interval,
- payload.startDate,
- payload.endDate
- ).map((item) => {
+ ? (series[key] ?? []).map((item) => {
return {
label: serieName,
count: item.count ? round(item.count) : null,
- date: new Date(item.date).toISOString(),
+ date: toDynamicISODateWithTZ(
+ item.date,
+ payload.startDate,
+ payload.interval
+ ),
};
})
: (series[key] ?? []).map((item) => ({
label: item.label,
count: item.count ? round(item.count) : null,
- date: new Date(item.date).toISOString(),
+ date: toDynamicISODateWithTZ(
+ item.date,
+ payload.startDate,
+ payload.interval
+ ),
}));
return {
@@ -310,8 +220,8 @@ export async function getChartData(payload: IGetChartDataInput) {
export function getDatesFromRange(range: IChartRange) {
if (range === '30min' || range === 'lastHour') {
const minutes = range === '30min' ? 30 : 60;
- const startDate = subMinutes(new Date(), minutes).toUTCString();
- const endDate = new Date().toUTCString();
+ const startDate = formatISO(subMinutes(new Date(), minutes));
+ const endDate = formatISO(new Date());
return {
startDate,
@@ -320,18 +230,22 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'today') {
- const startDate = startOfDay(new Date());
- const endDate = endOfDay(new Date());
+ // This is last 24 hours instead
+ // Makes it easier to handle timezones
+ // const startDate = startOfDay(new Date());
+ // const endDate = endOfDay(new Date());
+ const startDate = subDays(new Date(), 1);
+ const endDate = new Date();
return {
- startDate: startDate.toUTCString(),
- endDate: endDate.toUTCString(),
+ startDate: formatISO(startDate),
+ endDate: formatISO(endDate),
};
}
if (range === '7d') {
- const startDate = subDays(new Date(), 7).toUTCString();
- const endDate = new Date().toUTCString();
+ const startDate = formatISO(subDays(new Date(), 7));
+ const endDate = formatISO(new Date());
return {
startDate,
@@ -340,8 +254,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === '30d') {
- const startDate = subDays(new Date(), 30).toUTCString();
- const endDate = new Date().toUTCString();
+ const startDate = formatISO(subDays(new Date(), 30));
+ const endDate = formatISO(new Date());
return {
startDate,
@@ -350,8 +264,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'monthToDate') {
- const startDate = startOfMonth(new Date()).toUTCString();
- const endDate = new Date().toUTCString();
+ const startDate = formatISO(startOfMonth(new Date()));
+ const endDate = formatISO(new Date());
return {
startDate,
@@ -361,8 +275,8 @@ export function getDatesFromRange(range: IChartRange) {
if (range === 'lastMonth') {
const month = subMonths(new Date(), 1);
- const startDate = startOfMonth(month).toUTCString();
- const endDate = endOfDay(month).toUTCString();
+ const startDate = formatISO(startOfMonth(month));
+ const endDate = formatISO(endOfMonth(month));
return {
startDate,
@@ -371,8 +285,8 @@ export function getDatesFromRange(range: IChartRange) {
}
if (range === 'yearToDate') {
- const startDate = startOfYear(new Date()).toUTCString();
- const endDate = new Date().toUTCString();
+ const startDate = formatISO(startOfYear(new Date()));
+ const endDate = formatISO(new Date());
return {
startDate,
@@ -382,8 +296,8 @@ export function getDatesFromRange(range: IChartRange) {
if (range === 'lastYear') {
const year = subYears(new Date(), 1);
- const startDate = startOfYear(year).toUTCString();
- const endDate = endOfYear(year).toUTCString();
+ const startDate = formatISO(startOfYear(year));
+ const endDate = formatISO(endOfYear(year));
return {
startDate,
@@ -391,20 +305,9 @@ export function getDatesFromRange(range: IChartRange) {
};
}
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
- console.log('-------------------------------');
return {
- startDate: subDays(new Date(), 30).toISOString(),
- endDate: new Date().toISOString(),
+ startDate: formatISO(subDays(new Date(), 30)),
+ endDate: formatISO(new Date()),
};
}
@@ -421,16 +324,15 @@ export function getChartStartEndDate({
export function getChartPrevStartEndDate({
startDate,
endDate,
- range,
}: {
startDate: string;
endDate: string;
range: IChartRange;
}) {
- const diff = new Date(endDate).getTime() - new Date(startDate).getTime();
+ const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));
return {
- startDate: new Date(new Date(startDate).getTime() - diff).toISOString(),
- endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),
+ startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),
+ endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),
};
}