OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix timezones
Carl-Gerhard Lindesvärde4e6cc331673
diff --git a/packages/common/index.ts b/packages/common/index.tsindex 081f50fb9..b30cb315c 100644--- a/packages/common/index.ts+++ b/packages/common/index.ts@@ -6,3 +6,4 @@ export * from './src/names';export * from './src/string';export * from './src/math';export * from './src/slug';+export * from './src/fill-series';diff --git a/packages/common/package.json b/packages/common/package.jsonindex 45b357281..d682c25a7 100644--- a/packages/common/package.json+++ b/packages/common/package.json@@ -8,6 +8,7 @@"typecheck": "tsc --noEmit"},"dependencies": {+ "date-fns": "^3.3.1","mathjs": "^12.3.2","ramda": "^0.29.1","slugify": "^1.6.6",@@ -15,6 +16,7 @@"unique-names-generator": "^4.7.1"},"devDependencies": {+ "@openpanel/validation": "workspace:*","@openpanel/eslint-config": "workspace:*","@openpanel/prettier-config": "workspace:*","@openpanel/tsconfig": "workspace:*",diff --git a/packages/common/src/fill-series.ts b/packages/common/src/fill-series.tsnew file mode 100644index 000000000..0a7c6f198--- /dev/null+++ b/packages/common/src/fill-series.ts@@ -0,0 +1,93 @@+import {+ addDays,+ addHours,+ addMinutes,+ addMonths,+ format,+ parseISO,+ startOfDay,+ startOfHour,+ startOfMinute,+ startOfMonth,+} from 'date-fns';++import type { IInterval } from '@openpanel/validation';++// Define the data structure+interface DataEntry {+ label: string;+ count: number | null;+ date: string;+}++// Function to round down the date to the nearest interval+function roundDate(date: Date, interval: IInterval): Date {+ switch (interval) {+ case 'minute':+ return startOfMinute(date);+ case 'hour':+ return startOfHour(date);+ case 'day':+ return startOfDay(date);+ case 'month':+ return startOfMonth(date);+ default:+ return startOfMinute(date);+ }+}++// Function to complete the timeline for each label+export function completeTimeline(+ data: DataEntry[],+ _startDate: string,+ _endDate: string,+ interval: IInterval+) {+ const startDate = parseISO(_startDate);+ const endDate = parseISO(_endDate);+ // Group data by label+ const labelsMap = new Map<string, Map<string, number>>();+ data.forEach((entry) => {+ const roundedDate = roundDate(parseISO(entry.date), interval);+ const dateKey = format(roundedDate, 'yyyy-MM-dd HH:mm:ss');++ if (!labelsMap.has(entry.label)) {+ labelsMap.set(entry.label, new Map());+ }+ const labelData = labelsMap.get(entry.label);+ labelData?.set(dateKey, (labelData.get(dateKey) || 0) + (entry.count || 0));+ });++ // Complete the timeline for each label+ const result: Record<string, DataEntry[]> = {};+ labelsMap.forEach((counts, label) => {+ let currentDate = roundDate(startDate, interval);+ result[label] = [];+ while (currentDate <= endDate) {+ const dateKey = format(currentDate, 'yyyy-MM-dd HH:mm:ss');+ result[label]!.push({+ label: label,+ date: dateKey,+ count: counts.get(dateKey) || 0,+ });++ // Increment the current date based on the interval+ switch (interval) {+ case 'minute':+ currentDate = addMinutes(currentDate, 1);+ break;+ case 'hour':+ currentDate = addHours(currentDate, 1);+ break;+ case 'day':+ currentDate = addDays(currentDate, 1);+ break;+ case 'month':+ currentDate = addMonths(currentDate, 1);+ break;+ }+ }+ });++ return result;+}diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex be56c9339..4f0635d7a 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -23,35 +23,29 @@ export function getChartSql({sb.where = getEventFiltersWhereClause(event.filters);sb.where.projectId = `project_id = ${escape(projectId)}`;- let labelValue = escape('*');if (event.name !== '*') {- labelValue = `${escape(event.name)}`;- sb.select.label = `${labelValue} as label`;- sb.where.eventName = `name = ${labelValue}`;+ sb.select.label = `${escape(event.name)} as label`;+ sb.where.eventName = `name = ${escape(event.name)}`;} else {- sb.select.label = `${labelValue} as label`;+ sb.select.label = `'*' as label`;}sb.select.count = `count(*) as count`;switch (interval) {case 'minute': {sb.select.date = `toStartOfMinute(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMinute(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMinute(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'hour': {sb.select.date = `toStartOfHour(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfHour(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalHour(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'day': {sb.select.date = `toStartOfDay(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfDay(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalDay(1) INTERPOLATE ( label as ${labelValue} )`;break;}case 'month': {sb.select.date = `toStartOfMonth(toTimeZone(created_at, '${getTimezoneFromDateString(startDate)}')) as date`;- sb.orderBy.date = `date ASC WITH FILL FROM toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(startDate)}'), '${getTimezoneFromDateString(startDate)}')) TO toStartOfMonth(toTimeZone(toDateTime('${formatClickhouseDate(endDate)}'), '${getTimezoneFromDateString(startDate)}')) STEP toIntervalMonth(1) INTERPOLATE ( label as ${labelValue} )`;break;}}diff --git a/packages/trpc/src/routers/chart.helpers.ts b/packages/trpc/src/routers/chart.helpers.tsindex 44a4d9f3a..94b68127d 100644--- a/packages/trpc/src/routers/chart.helpers.ts+++ b/packages/trpc/src/routers/chart.helpers.ts@@ -17,7 +17,7 @@ import * as mathjs from 'mathjs';import { repeat, reverse, sort } from 'ramda';import { escape } from 'sqlstring';-import { round } from '@openpanel/common';+import { completeTimeline, round } from '@openpanel/common';import { alphabetIds, NOT_SET_VALUE } from '@openpanel/constants';import {chQuery,@@ -133,88 +133,59 @@ const toDynamicISODateWithTZ = (// It will be converted to the correct timezone on the clientreturn date.replace(' ', 'T');}- return `${date}T00:00:00`;+ return `${date}T00:00:00Z`;};export async function getChartData(payload: IGetChartDataInput) {- let result = await chQuery<ResultItem>(getChartSql(payload));-- if (result.length === 0 && payload.breakdowns.length > 0) {- result = await chQuery<ResultItem>(- getChartSql({- ...payload,- breakdowns: [],- })- );+ async function getSeries() {+ const result = await chQuery<ResultItem>(getChartSql(payload));+ if (result.length === 0 && payload.breakdowns.length > 0) {+ return await chQuery<ResultItem>(+ getChartSql({+ ...payload,+ breakdowns: [],+ })+ );+ }+ return result;}- // group by sql label- const series = result.reduce(- (acc, item) => {- // If we fill empty spots in the timeline (clickhouse) we wont get a label back- // take the event name as label- if (!item.label && item.count === 0) {- item.label = payload.event.name;- }-- const label = item.label?.trim() || NOT_SET_VALUE;- // item.label can be null when using breakdowns on a property- // that doesn't exist on all events- if (label) {- if (acc[label]) {- acc[label]?.push(item);- } else {- acc[label] = [item];- }- }-- return {- ...acc,- };- },- {} as Record<string, ResultItem[]>- );+ return getSeries()+ .then((data) =>+ completeTimeline(+ data.map((item) => {+ const label = item.label?.trim() || NOT_SET_VALUE;- return Object.keys(series).map((key) => {- // If we have breakdowns, we want to use the breakdown key as the legend- // But only if it successfully broke it down, otherwise we use the getEventLabel- const isBreakdown =- payload.breakdowns.length && !alphabetIds.includes(key as 'A');- const serieName = isBreakdown ? key : getEventLegend(payload.event);- const data =- payload.chartType === 'area' ||- payload.chartType === 'linear' ||- payload.chartType === 'histogram' ||- payload.chartType === 'metric' ||- payload.chartType === 'pie' ||- payload.chartType === 'bar'- ? (series[key] ?? []).map((item) => {- return {- label: serieName,- count: item.count ? round(item.count) : null,- date: toDynamicISODateWithTZ(- item.date,- payload.startDate,- payload.interval- ),- };- })- : (series[key] ?? []).map((item) => ({- label: item.label,+ return {+ ...item,count: item.count ? round(item.count) : null,+ label,+ };+ }),+ payload.startDate,+ payload.endDate,+ payload.interval+ )+ )+ .then((series) => {+ return Object.keys(series).map((label) => {+ const isBreakdown =+ payload.breakdowns.length && !alphabetIds.includes(label as 'A');+ const serieLabel = isBreakdown ? label : getEventLegend(payload.event);+ return {+ name: serieLabel,+ event: payload.event,+ data: series[label]!.map((item) => ({+ ...item,date: toDynamicISODateWithTZ(item.date,payload.startDate,payload.interval),- }));-- return {- name: serieName,- event: payload.event,- data,- };- });+ })),+ };+ });+ });}export function getDatesFromRange(range: IChartRange) {diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex e77aeb52f..c32736f08 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -184,7 +184,7 @@ export const chartRouter = createTRPCRouter({}),// TODO: Make this private- chart: publicProcedure.input(zChartInput).query(async ({ input, ctx }) => {+ chart: publicProcedure.input(zChartInput).query(async ({ input }) => {const currentPeriod = getChartStartEndDate(input);const previousPeriod = getChartPrevStartEndDate({range: input.range,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d35fcfaa2..6a9b5c6e6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -763,6 +763,9 @@ importers:packages/common:dependencies:+ date-fns:+ specifier: ^3.3.1+ version: 3.3.1mathjs:specifier: ^12.3.2version: 12.3.2@@ -788,6 +791,9 @@ importers:'@openpanel/tsconfig':specifier: workspace:*version: link:../../tooling/typescript+ '@openpanel/validation':+ specifier: workspace:*+ version: link:../validation'@types/node':specifier: ^18.16.0version: 18.19.17