OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
add better support for timezones
Carl-Gerhard Lindesvärd33e92a0da0f3
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 1f98d79ee..2d698e35c 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -56,8 +56,8 @@ const startServer = async () => {onError(error: unknown) {if (error instanceof Error) {logger.error(error, error.message);- } else {- logger.error(error, 'Unknown error trpc error');+ } else if (error && typeof error === 'object' && 'error' in error) {+ logger.error(error.error, 'Unknown error trpc error');}},} satisfies FastifyTRPCPluginOptions<AppRouter>['trpcOptions'],diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsxindex dbc335d25..6722fda93 100644--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx+++ b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/overview-sticky-header.tsx@@ -2,6 +2,7 @@import { useOverviewOptions } from '@/components/overview/useOverviewOptions';import { TimeWindowPicker } from '@/components/time-window-picker';+import { endOfDay, formatISO, startOfDay } from 'date-fns';export function OverviewReportRange() {const { range, setRange, setStartDate, setEndDate, endDate, startDate } =@@ -11,8 +12,14 @@ export function OverviewReportRange() {<TimeWindowPickeronChange={setRange}value={range}- onStartDateChange={(date) => setStartDate(date)}- onEndDateChange={(date) => setEndDate(date)}+ onStartDateChange={(date) => {+ const d = formatISO(startOfDay(new Date(date)));+ setStartDate(d);+ }}+ onEndDateChange={(date) => {+ const d = formatISO(endOfDay(new Date(date)));+ setEndDate(d);+ }}endDate={endDate}startDate={startDate}/>diff --git a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx b/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsxdeleted file mode 100644index 9cf2cf500..000000000--- a/apps/dashboard/src/app/(app)/[organizationSlug]/[projectId]/reports/loading.tsx+++ /dev/null@@ -1,3 +0,0 @@-import FullPageLoadingState from '@/components/full-page-loading-state';--export default FullPageLoadingState;diff --git a/apps/dashboard/src/components/report/reportSlice.ts b/apps/dashboard/src/components/report/reportSlice.tsindex 0616a3316..b63211286 100644--- a/apps/dashboard/src/components/report/reportSlice.ts+++ b/apps/dashboard/src/components/report/reportSlice.ts@@ -1,6 +1,12 @@import { createSlice } from '@reduxjs/toolkit';import type { PayloadAction } from '@reduxjs/toolkit';-import { isSameDay, isSameMonth } from 'date-fns';+import {+ endOfDay,+ formatISO,+ isSameDay,+ isSameMonth,+ startOfDay,+} from 'date-fns';import {alphabetIds,@@ -188,8 +194,8 @@ export const reportSlice = createSlice({}>) => {state.dirty = true;- state.startDate = action.payload.startDate;- state.endDate = action.payload.endDate;+ state.startDate = formatISO(startOfDay(action.payload.startDate));+ state.endDate = formatISO(endOfDay(action.payload.endDate));if (isSameDay(state.startDate, state.endDate)) {state.interval = 'hour';@@ -203,7 +209,7 @@ export const reportSlice = createSlice({// Date rangechangeStartDate: (state, action: PayloadAction<string>) => {state.dirty = true;- state.startDate = action.payload;+ state.startDate = formatISO(startOfDay(action.payload));const interval = getDefaultIntervalByDates(state.startDate,@@ -217,7 +223,7 @@ export const reportSlice = createSlice({// Date rangechangeEndDate: (state, action: PayloadAction<string>) => {state.dirty = true;- state.endDate = action.payload;+ state.endDate = formatISO(endOfDay(action.payload));const interval = getDefaultIntervalByDates(state.startDate,diff --git a/packages/common/src/date.ts b/packages/common/src/date.tsindex b2bbdf8dd..2f520cc29 100644--- a/packages/common/src/date.ts+++ b/packages/common/src/date.ts@@ -5,3 +5,41 @@ export function getTime(date: string | number | Date) {export function toISOString(date: string | number | Date) {return new Date(date).toISOString();}++export function getTimezoneFromDateString(_date: string) {+ const mapper: Record<string, string> = {+ '+00:00': 'UTC',+ '+01:00': 'Europe/Paris',+ '+02:00': 'Europe/Stockholm',+ '+03:00': 'Europe/Moscow',+ '+04:00': 'Asia/Dubai',+ '+05:00': 'Asia/Karachi',+ '+06:00': 'Asia/Dhaka',+ '+07:00': 'Asia/Bangkok',+ '+08:00': 'Asia/Shanghai',+ '+09:00': 'Asia/Tokyo',+ '+10:00': 'Australia/Sydney',+ '+11:00': 'Pacific/Noumea',+ '+12:00': 'Pacific/Fiji',+ '-02:00': 'America/Noronha',+ '-03:00': 'America/Sao_Paulo',+ '-04:00': 'America/Santiago',+ '-05:00': 'America/Bogota',+ '-06:00': 'America/Mexico_City',+ '-07:00': 'America/Phoenix',+ '-08:00': 'America/Los_Angeles',+ '-09:00': 'America/Anchorage',+ '-10:00': 'Pacific/Honolulu',+ '-11:00': 'Pacific/Midway',+ '-12:00': 'Pacific/Tarawa',+ };++ const defaultTimezone = 'UTC';++ const match = _date.match(/([+-][0-9]{2}):([0-9]{2})$/)?.[0];+ if (match) {+ return mapper[match] ?? defaultTimezone;+ }++ return defaultTimezone;+}diff --git a/packages/constants/index.ts b/packages/constants/index.tsindex c2bf653fe..acd51f948 100644--- a/packages/constants/index.ts+++ b/packages/constants/index.ts@@ -15,7 +15,7 @@ export const timeWindows = {},today: {key: 'today',- label: 'Today',+ label: '24 hours',shortcut: 'D',},'7d': {diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 2d3064cd6..3966c7551 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -15,6 +15,7 @@ export const ch = createClient({export async function chQueryWithMeta<T extends Record<string, any>>(query: string): Promise<ResponseJSON<T>> {+ console.log('Query:', query);const start = Date.now();const res = await ch.query({query,@@ -39,7 +40,6 @@ export async function chQueryWithMeta<T extends Record<string, any>>(console.log(`Clickhouse query took ${response.statistics?.elapsed}ms`);console.log(`chQuery took ${Date.now() - start}ms`);- console.log('Query:', query);return response;}@@ -52,7 +52,11 @@ export async function chQuery<T extends Record<string, any>>(export function formatClickhouseDate(_date: Date | string) {const date = typeof _date === 'string' ? new Date(_date) : _date;- return date.toISOString().replace('T', ' ').replace(/Z+$/, '');+ return date+ .toISOString()+ .replace('T', ' ')+ .replace(/Z+$/, '')+ .replace(/\.[0-9]+$/, '');}export function convertClickhouseDateToJs(date: string) {diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex a49dd787e..be56c9339 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -1,5 +1,6 @@import { escape } from 'sqlstring';+import { getTimezoneFromDateString } from '@openpanel/common';import type {IChartEventFilter,IGetChartDataInput,@@ -21,32 +22,40 @@ export function getChartSql({sb.where = getEventFiltersWhereClause(event.filters);sb.where.projectId = `project_id = ${escape(projectId)}`;++ let labelValue = escape('*');if (event.name !== '*') {- sb.select.label = `${escape(event.name)} as label`;- sb.where.eventName = `name = ${escape(event.name)}`;+ labelValue = `${escape(event.name)}`;+ sb.select.label = `${labelValue} as label`;+ sb.where.eventName = `name = ${labelValue}`;+ } else {+ sb.select.label = `${labelValue} as label`;}sb.select.count = `count(*) as count`;switch (interval) {case 'minute': {- sb.select.date = `toStartOfMinute(created_at) as date`;+ sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'hour': {- sb.select.date = `toStartOfHour(created_at) as date`;+ sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'day': {- sb.select.date = `toStartOfDay(created_at) as date`;+ sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'month': {- sb.select.date = `toStartOfMonth(created_at) as date`;+ sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;+ sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;break;}}sb.groupBy.date = 'date';- sb.orderBy.date = 'date ASC';if (startDate) {sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.tsindex f2f98c610..44a4d9f3a 100644--- a/packages/trpc/src/routers/chart.helpers.ts+++ b/packages/trpc/src/routers/chart.helpers.ts@@ -1,10 +1,14 @@import {+ differenceInMilliseconds,endOfDay,+ endOfMonth,endOfYear,+ formatISO,startOfDay,startOfMonth,startOfYear,subDays,+ subMilliseconds,subMinutes,subMonths,subYears,@@ -43,129 +47,6 @@ function getEventLegend(event: IChartEvent) {return event.displayName ?? `${event.name} (${event.id})`;}-function fillEmptySpotsInTimeline(- items: ResultItem[],- interval: IInterval,- startDate: string,- endDate: string-) {- const result = [];- const clonedStartDate = new Date(startDate);- const clonedEndDate = new Date(endDate);- const today = new Date();-- if (interval === 'minute') {- clonedStartDate.setUTCSeconds(0, 0);- clonedEndDate.setUTCMinutes(clonedEndDate.getUTCMinutes() + 1, 0, 0);- } else if (interval === 'hour') {- clonedStartDate.setUTCMinutes(0, 0, 0);- clonedEndDate.setUTCMinutes(0, 0, 0);- } else {- clonedStartDate.setUTCHours(0, 0, 0, 0);- clonedEndDate.setUTCHours(0, 0, 0, 0);- }-- if (interval === 'month') {- clonedStartDate.setUTCDate(1);- clonedEndDate.setUTCDate(1);- }-- // Force if interval is month and the start date is the same month as today- const shouldForce = () =>- interval === 'month' &&- clonedStartDate.getUTCFullYear() === today.getUTCFullYear() &&- clonedStartDate.getUTCMonth() === today.getUTCMonth();- let prev = undefined;- while (- shouldForce() ||- clonedStartDate.getTime() <= clonedEndDate.getTime()- ) {- if (prev === clonedStartDate.getTime()) {- break;- }- prev = clonedStartDate.getTime();-- const getYear = (date: Date) => date.getUTCFullYear();- const getMonth = (date: Date) => date.getUTCMonth();- const getDay = (date: Date) => date.getUTCDate();- const getHour = (date: Date) => date.getUTCHours();- const getMinute = (date: Date) => date.getUTCMinutes();-- const item = items.find((item) => {- const date = convertClickhouseDateToJs(item.date);-- if (interval === 'month') {- return (- getYear(date) === getYear(clonedStartDate) &&- getMonth(date) === getMonth(clonedStartDate)- );- }- if (interval === 'day') {- return (- getYear(date) === getYear(clonedStartDate) &&- getMonth(date) === getMonth(clonedStartDate) &&- getDay(date) === getDay(clonedStartDate)- );- }- if (interval === 'hour') {- return (- getYear(date) === getYear(clonedStartDate) &&- getMonth(date) === getMonth(clonedStartDate) &&- getDay(date) === getDay(clonedStartDate) &&- getHour(date) === getHour(clonedStartDate)- );- }- if (interval === 'minute') {- return (- getYear(date) === getYear(clonedStartDate) &&- getMonth(date) === getMonth(clonedStartDate) &&- getDay(date) === getDay(clonedStartDate) &&- getHour(date) === getHour(clonedStartDate) &&- getMinute(date) === getMinute(clonedStartDate)- );- }-- return false;- });-- if (item) {- result.push({- ...item,- date: clonedStartDate.toISOString(),- });- } else {- result.push({- date: clonedStartDate.toISOString(),- count: 0,- label: null,- });- }-- switch (interval) {- case 'day': {- clonedStartDate.setUTCDate(clonedStartDate.getUTCDate() + 1);- break;- }- case 'hour': {- clonedStartDate.setUTCHours(clonedStartDate.getUTCHours() + 1);- break;- }- case 'minute': {- clonedStartDate.setUTCMinutes(clonedStartDate.getUTCMinutes() + 1);- break;- }- case 'month': {- clonedStartDate.setUTCMonth(clonedStartDate.getUTCMonth() + 1);- break;- }- }- }-- return sort(function (a, b) {- return new Date(a.date).getTime() - new Date(b.date).getTime();- }, result);-}-export function withFormula({ formula, events }: IChartInput,series: GetChartDataResult@@ -235,6 +116,26 @@ export function withFormula(];}+const toDynamicISODateWithTZ = (+ date: string,+ blueprint: string,+ interval: IInterval+) => {+ // If we have a space in the date we know it's a date with time+ if (date.includes(' ')) {+ // If interval is minutes we need to convert the timezone to what timezone is used (either on client or the server)+ // - We use timezone from server if its a predefined range (yearToDate, lastYear, etc.)+ // - We use timezone from client if its a custom range+ if (interval === 'minute' || interval === 'hour') {+ return date.replace(' ', 'T') + blueprint.slice(-6);+ }+ // Otherwise we just return without the timezone+ // It will be converted to the correct timezone on the client+ return date.replace(' ', 'T');+ }+ return `${date}T00:00:00`;+};+export async function getChartData(payload: IGetChartDataInput) {let result = await chQuery<ResultItem>(getChartSql(payload));@@ -250,9 +151,15 @@ export async function getChartData(payload: IGetChartDataInput) {// group by sql labelconst series = result.reduce((acc, item) => {+ // If we fill empty spots in the timeline (clickhouse) we wont get a label back+ // take the event name as label+ if (!item.label && item.count === 0) {+ item.label = payload.event.name;+ }++ const label = item.label?.trim() || NOT_SET_VALUE;// item.label can be null when using breakdowns on a property// that doesn't exist on all events- const label = item.label?.trim() || NOT_SET_VALUE;if (label) {if (acc[label]) {acc[label]?.push(item);@@ -281,22 +188,25 @@ export async function getChartData(payload: IGetChartDataInput) {payload.chartType === 'metric' ||payload.chartType === 'pie' ||payload.chartType === 'bar'- ? fillEmptySpotsInTimeline(- series[key] ?? [],- payload.interval,- payload.startDate,- payload.endDate- ).map((item) => {+ ? (series[key] ?? []).map((item) => {return {label: serieName,count: item.count ? round(item.count) : null,- date: new Date(item.date).toISOString(),+ date: toDynamicISODateWithTZ(+ item.date,+ payload.startDate,+ payload.interval+ ),};}): (series[key] ?? []).map((item) => ({label: item.label,count: item.count ? round(item.count) : null,- date: new Date(item.date).toISOString(),+ date: toDynamicISODateWithTZ(+ item.date,+ payload.startDate,+ payload.interval+ ),}));return {@@ -310,8 +220,8 @@ export async function getChartData(payload: IGetChartDataInput) {export function getDatesFromRange(range: IChartRange) {if (range === '30min' || range === 'lastHour') {const minutes = range === '30min' ? 30 : 60;- const startDate = subMinutes(new Date(), minutes).toUTCString();- const endDate = new Date().toUTCString();+ const startDate = formatISO(subMinutes(new Date(), minutes));+ const endDate = formatISO(new Date());return {startDate,@@ -320,18 +230,22 @@ export function getDatesFromRange(range: IChartRange) {}if (range === 'today') {- const startDate = startOfDay(new Date());- const endDate = endOfDay(new Date());+ // This is last 24 hours instead+ // Makes it easier to handle timezones+ // const startDate = startOfDay(new Date());+ // const endDate = endOfDay(new Date());+ const startDate = subDays(new Date(), 1);+ const endDate = new Date();return {- startDate: startDate.toUTCString(),- endDate: endDate.toUTCString(),+ startDate: formatISO(startDate),+ endDate: formatISO(endDate),};}if (range === '7d') {- const startDate = subDays(new Date(), 7).toUTCString();- const endDate = new Date().toUTCString();+ const startDate = formatISO(subDays(new Date(), 7));+ const endDate = formatISO(new Date());return {startDate,@@ -340,8 +254,8 @@ export function getDatesFromRange(range: IChartRange) {}if (range === '30d') {- const startDate = subDays(new Date(), 30).toUTCString();- const endDate = new Date().toUTCString();+ const startDate = formatISO(subDays(new Date(), 30));+ const endDate = formatISO(new Date());return {startDate,@@ -350,8 +264,8 @@ export function getDatesFromRange(range: IChartRange) {}if (range === 'monthToDate') {- const startDate = startOfMonth(new Date()).toUTCString();- const endDate = new Date().toUTCString();+ const startDate = formatISO(startOfMonth(new Date()));+ const endDate = formatISO(new Date());return {startDate,@@ -361,8 +275,8 @@ export function getDatesFromRange(range: IChartRange) {if (range === 'lastMonth') {const month = subMonths(new Date(), 1);- const startDate = startOfMonth(month).toUTCString();- const endDate = endOfDay(month).toUTCString();+ const startDate = formatISO(startOfMonth(month));+ const endDate = formatISO(endOfMonth(month));return {startDate,@@ -371,8 +285,8 @@ export function getDatesFromRange(range: IChartRange) {}if (range === 'yearToDate') {- const startDate = startOfYear(new Date()).toUTCString();- const endDate = new Date().toUTCString();+ const startDate = formatISO(startOfYear(new Date()));+ const endDate = formatISO(new Date());return {startDate,@@ -382,8 +296,8 @@ export function getDatesFromRange(range: IChartRange) {if (range === 'lastYear') {const year = subYears(new Date(), 1);- const startDate = startOfYear(year).toUTCString();- const endDate = endOfYear(year).toUTCString();+ const startDate = formatISO(startOfYear(year));+ const endDate = formatISO(endOfYear(year));return {startDate,@@ -391,20 +305,9 @@ export function getDatesFromRange(range: IChartRange) {};}- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');- console.log('-------------------------------');return {- startDate: subDays(new Date(), 30).toISOString(),- endDate: new Date().toISOString(),+ startDate: formatISO(subDays(new Date(), 30)),+ endDate: formatISO(new Date()),};}@@ -421,16 +324,15 @@ export function getChartStartEndDate({export function getChartPrevStartEndDate({startDate,endDate,- range,}: {startDate: string;endDate: string;range: IChartRange;}) {- const diff = new Date(endDate).getTime() - new Date(startDate).getTime();+ const diff = differenceInMilliseconds(new Date(endDate), new Date(startDate));return {- startDate: new Date(new Date(startDate).getTime() - diff).toISOString(),- endDate: new Date(new Date(endDate).getTime() - diff).toISOString(),+ startDate: formatISO(subMilliseconds(new Date(startDate), diff - 1)),+ endDate: formatISO(subMilliseconds(new Date(endDate), diff - 1)),};}