Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix timezones

Carl-Gerhard Lindesvärde4e6cc331673
diff --git a/packages/common/index.ts b/packages/common/index.ts
index 081f50fb9..b30cb315c 100644
--- a/packages/common/index.ts
+++ b/packages/common/index.ts
@@ -6,3 +6,4 @@ export * from './src/names';
export * from './src/string';
export * from './src/math';
export * from './src/slug';
+export * from './src/fill-series';
diff --git a/packages/common/package.json b/packages/common/package.json
index 45b357281..d682c25a7 100644
--- a/packages/common/package.json
+++ b/packages/common/package.json
@@ -8,6 +8,7 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
+ "date-fns": "^3.3.1",
"mathjs": "^12.3.2",
"ramda": "^0.29.1",
"slugify": "^1.6.6",
@@ -15,6 +16,7 @@
"unique-names-generator": "^4.7.1"
},
"devDependencies": {
+ "@openpanel/validation": "workspace:*",
"@openpanel/eslint-config": "workspace:*",
"@openpanel/prettier-config": "workspace:*",
"@openpanel/tsconfig": "workspace:*",
diff --git a/packages/common/src/fill-series.ts b/packages/common/src/fill-series.ts
new file mode 100644
index 000000000..0a7c6f198
--- /dev/null
+++ b/packages/common/src/fill-series.ts
@@ -0,0 +1,93 @@
+import {
+ addDays,
+ addHours,
+ addMinutes,
+ addMonths,
+ format,
+ parseISO,
+ startOfDay,
+ startOfHour,
+ startOfMinute,
+ startOfMonth,
+} from 'date-fns';
+
+import type { IInterval } from '@openpanel/validation';
+
+// Define the data structure
+interface DataEntry {
+ label: string;
+ count: number | null;
+ date: string;
+}
+
+// Function to round down the date to the nearest interval
+function roundDate(date: Date, interval: IInterval): Date {
+ switch (interval) {
+ case 'minute':
+ return startOfMinute(date);
+ case 'hour':
+ return startOfHour(date);
+ case 'day':
+ return startOfDay(date);
+ case 'month':
+ return startOfMonth(date);
+ default:
+ return startOfMinute(date);
+ }
+}
+
+// Function to complete the timeline for each label
+export function completeTimeline(
+ data: DataEntry[],
+ _startDate: string,
+ _endDate: string,
+ interval: IInterval
+) {
+ const startDate = parseISO(_startDate);
+ const endDate = parseISO(_endDate);
+ // Group data by label
+ const labelsMap = new Map<string, Map<string, number>>();
+ data.forEach((entry) => {
+ const roundedDate = roundDate(parseISO(entry.date), interval);
+ const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');
+
+ if (!labelsMap.has(entry.label)) {
+ labelsMap.set(entry.label, new Map());
+ }
+ const labelData = labelsMap.get(entry.label);
+ labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));
+ });
+
+ // Complete the timeline for each label
+ const result: Record<string, DataEntry[]> = {};
+ labelsMap.forEach((counts, label) => {
+ let currentDate = roundDate(startDate, interval);
+ result[label] = [];
+ while (currentDate <= endDate) {
+ const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss');
+ result[label]!.push({
+ label: label,
+ date: dateKey,
+ count: counts.get(dateKey) || 0,
+ });
+
+ // Increment the current date based on the interval
+ switch (interval) {
+ case 'minute':
+ currentDate = addMinutes(currentDate, 1);
+ break;
+ case 'hour':
+ currentDate = addHours(currentDate, 1);
+ break;
+ case 'day':
+ currentDate = addDays(currentDate, 1);
+ break;
+ case 'month':
+ currentDate = addMonths(currentDate, 1);
+ break;
+ }
+ }
+ });
+
+ return result;
+}
diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts
index be56c9339..4f0635d7a 100644
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -23,35 +23,29 @@ export function getChartSql({
sb.where = getEventFiltersWhereClause(event.filters);
sb.where.projectId = `project_id = ${escape(projectId)}`;
- let labelValue = escape('*');
if (event.name !== '*') {
- labelValue = `${escape(event.name)}`;
- sb.select.label = `${labelValue} as label`;
- sb.where.eventName = `name = ${labelValue}`;
+ sb.select.label = `${escape(event.name)} as label`;
+ sb.where.eventName = `name = ${escape(event.name)}`;
} else {
- sb.select.label = `${labelValue} as label`;
+ sb.select.label = `'*' as label`;
}
sb.select.count = `count(*) as count`;
switch (interval) {
case 'minute': {
sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'hour': {
sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'day': {
sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
case 'month': {
sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;
- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;
break;
}
}
diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.ts
index 44a4d9f3a..94b68127d 100644
--- a/packages/trpc/src/routers/chart.helpers.ts
+++ b/packages/trpc/src/routers/chart.helpers.ts
@@ -17,7 +17,7 @@ import * as mathjs from 'mathjs';
import { repeat, reverse, sort } from 'ramda';
import { escape } from 'sqlstring';
-import { round } from '@openpanel/common';
+import { completeTimeline, round } from '@openpanel/common';
import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';
import {
chQuery,
@@ -133,88 +133,59 @@ const toDynamicISODateWithTZ = (
// It will be converted to the correct timezone on the client
return date.replace(' ', 'T');
}
- return `${date}T00:00:00`;
+ return `${date}T00:00:00Z`;
};
export async function getChartData(payload: IGetChartDataInput) {
- let result = await chQuery<ResultItem>(getChartSql(payload));
-
- if (result.length === 0 && payload.breakdowns.length > 0) {
- result = await chQuery<ResultItem>(
- getChartSql({
- ...payload,
- breakdowns: [],
- })
- );
+ async function getSeries() {
+ const result = await chQuery<ResultItem>(getChartSql(payload));
+ if (result.length === 0 && payload.breakdowns.length > 0) {
+ return await chQuery<ResultItem>(
+ getChartSql({
+ ...payload,
+ breakdowns: [],
+ })
+ );
+ }
+ return result;
}
- // group by sql label
- const series = result.reduce(
- (acc, item) => {
- // If we fill empty spots in the timeline (clickhouse) we wont get a label back
- // take the event name as label
- if (!item.label && item.count === 0) {
- item.label = payload.event.name;
- }
-
- const label = item.label?.trim() || NOT_SET_VALUE;
- // item.label can be null when using breakdowns on a property
- // that doesn't exist on all events
- if (label) {
- if (acc[label]) {
- acc[label]?.push(item);
- } else {
- acc[label] = [item];
- }
- }
-
- return {
- ...acc,
- };
- },
- {} as Record<string, ResultItem[]>
- );
+ return getSeries()
+ .then((data) =>
+ completeTimeline(
+ data.map((item) => {
+ const label = item.label?.trim() || NOT_SET_VALUE;
- return Object.keys(series).map((key) => {
- // If we have breakdowns, we want to use the breakdown key as the legend
- // But only if it successfully broke it down, otherwise we use the getEventLabel
- const isBreakdown =
- payload.breakdowns.length && !alphabetIds.includes(key as 'A');
- const serieName = isBreakdown ? key : getEventLegend(payload.event);
- const data =
- payload.chartType === 'area' ||
- payload.chartType === 'linear' ||
- payload.chartType === 'histogram' ||
- payload.chartType === 'metric' ||
- payload.chartType === 'pie' ||
- payload.chartType === 'bar'
- ? (series[key] ?? []).map((item) => {
- return {
- label: serieName,
- count: item.count ? round(item.count) : null,
- date: toDynamicISODateWithTZ(
- item.date,
- payload.startDate,
- payload.interval
- ),
- };
- })
- : (series[key] ?? []).map((item) => ({
- label: item.label,
+ return {
+ ...item,
count: item.count ? round(item.count) : null,
+ label,
+ };
+ }),
+ payload.startDate,
+ payload.endDate,
+ payload.interval
+ )
+ )
+ .then((series) => {
+ return Object.keys(series).map((label) => {
+ const isBreakdown =
+ payload.breakdowns.length && !alphabetIds.includes(label as 'A');
+ const serieLabel = isBreakdown ? label : getEventLegend(payload.event);
+ return {
+ name: serieLabel,
+ event: payload.event,
+ data: series[label]!.map((item) => ({
+ ...item,
date: toDynamicISODateWithTZ(
item.date,
payload.startDate,
payload.interval
),
- }));
-
- return {
- name: serieName,
- event: payload.event,
- data,
- };
- });
+ })),
+ };
+ });
+ });
}
export function getDatesFromRange(range: IChartRange) {
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index e77aeb52f..c32736f08 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -184,7 +184,7 @@ export const chartRouter = createTRPCRouter({
}),
// TODO: Make this private
- chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {
+ chart: publicProcedure.input(zChartInput).query(async ({ input }) => {
const currentPeriod = getChartStartEndDate(input);
const previousPeriod = getChartPrevStartEndDate({
range: input.range,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d35fcfaa2..6a9b5c6e6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -763,6 +763,9 @@ importers:
packages/common:
dependencies:
+ date-fns:
+ specifier: ^3.3.1
+ version: 3.3.1
mathjs:
specifier: ^12.3.2
version: 12.3.2
@@ -788,6 +791,9 @@ importers:
'@openpanel/tsconfig':
specifier: workspace:*
version: link:../../tooling/typescript
+ '@openpanel/validation':
+ specifier: workspace:*
+ version: link:../validation
'@types/node':
specifier: ^18.16.0
version: 18.19.17