OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
better handling dates in clickhouse
Carl-Gerhard Lindesvärd2be2ff3e12f2
diff --git a/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx b/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsxindex d5da4359d..4a4f14edd 100644--- a/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx+++ b/apps/dashboard/src/app/(onboarding)/skip-onboarding.tsx@@ -29,7 +29,7 @@ const SkipOnboarding = () => {text: 'Are you sure you want to skip onboarding? Since you do not have any projects, you will be logged out.',onConfirm() {auth.signOut();- router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL);+ router.replace(process.env.NEXT_PUBLIC_CLERK_SIGN_IN_URL!);},});}diff --git a/apps/dashboard/src/app/providers.tsx b/apps/dashboard/src/app/providers.tsxindex 918add317..ea7857dd0 100644--- a/apps/dashboard/src/app/providers.tsx+++ b/apps/dashboard/src/app/providers.tsx@@ -62,14 +62,14 @@ function AllProviders({ children }: { children: React.ReactNode }) {defaultTheme="light"disableTransitionOnChange>- {process.env.NEXT_PUBLIC_OP_CLIENT_ID && (- <OpenPanelComponent- clientId={process.env.NEXT_PUBLIC_OP_CLIENT_ID}- trackScreenViews- trackOutgoingLinks- trackAttributes- />- )}+ <OpenPanelComponent+ apiUrl="http://localhost:3333"+ clientId={'75479ffd-645b-43d2-a33a-2111a6f5ee00'}+ trackScreenViews+ trackOutgoingLinks+ trackAttributes+ />+<ReduxProvider store={storeRef.current}><api.Provider client={trpcClient} queryClient={queryClient}><QueryClientProvider client={queryClient}>diff --git a/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx b/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsxindex 4da8cd889..0ac84a32f 100644--- a/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx+++ b/apps/dashboard/src/components/overview/filters/overview-filters-drawer-content.tsx@@ -20,6 +20,8 @@ import type {IChartEventFilterValue,} from '@openpanel/validation';+import { useOverviewOptions } from '../useOverviewOptions';+export interface OverviewFiltersDrawerContentProps {projectId: string;nuqsOptions?: NuqsOptions;@@ -33,10 +35,11 @@ export function OverviewFiltersDrawerContent({enableEventsFilter,mode,}: OverviewFiltersDrawerContentProps) {+ const { interval, range } = useOverviewOptions();const [filters, setFilter] = useEventQueryFilters(nuqsOptions);const [event, setEvent] = useEventQueryNamesFilter(nuqsOptions);- const eventNames = useEventNames(projectId);- const eventProperties = useEventProperties(projectId);+ const eventNames = useEventNames({ projectId, interval, range });+ const eventProperties = useEventProperties({ projectId, interval, range });const profileProperties = useProfileProperties(projectId);const properties = mode === 'events' ? eventProperties : profileProperties;@@ -113,11 +116,14 @@ export function FilterOptionEvent({operator: IChartEventFilterOperator) => void;}) {- const values = useEventValues(+ const { interval, range } = useOverviewOptions();+ const values = useEventValues({projectId,- filter.name === 'path' ? 'screen_view' : 'session_start',- filter.name- );+ event: filter.name === 'path' ? 'screen_view' : 'session_start',+ property: filter.name,+ interval,+ range,+ });return (<div className="flex items-center gap-2">diff --git a/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx b/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsxindex a021774ad..8d8ecbd44 100644--- a/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx+++ b/apps/dashboard/src/components/report/sidebar/EventPropertiesCombobox.tsx@@ -1,6 +1,6 @@import { Combobox } from '@/components/ui/combobox';import { useAppParams } from '@/hooks/useAppParams';-import { useDispatch } from '@/redux';+import { useDispatch, useSelector } from '@/redux';import { api } from '@/trpc/client';import { cn } from '@/utils/cn';import { DatabaseIcon } from 'lucide-react';@@ -18,11 +18,14 @@ export function EventPropertiesCombobox({}: EventPropertiesComboboxProps) {const dispatch = useDispatch();const { projectId } = useAppParams();-+ const range = useSelector((state) => state.report.range);+ const interval = useSelector((state) => state.report.interval);const query = api.chart.properties.useQuery({event: event.name,projectId,+ range,+ interval,},{enabled: !!event.name,diff --git a/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx b/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsxindex 3be40b1ea..0d1f54189 100644--- a/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx+++ b/apps/dashboard/src/components/report/sidebar/ReportBreakdowns.tsx@@ -16,9 +16,14 @@ import type { ReportEventMoreProps } from './ReportEventMore';export function ReportBreakdowns() {const { projectId } = useAppParams();const selectedBreakdowns = useSelector((state) => state.report.breakdowns);+ const interval = useSelector((state) => state.report.interval);+ const range = useSelector((state) => state.report.range);+const dispatch = useDispatch();const propertiesQuery = api.chart.properties.useQuery({projectId,+ range,+ interval,});const propertiesCombobox = (propertiesQuery.data ?? []).map((item) => ({value: item,diff --git a/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx b/apps/dashboard/src/components/report/sidebar/ReportEvents.tsxindex ce1989e76..42c2e9a37 100644--- a/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx+++ b/apps/dashboard/src/components/report/sidebar/ReportEvents.tsx@@ -29,13 +29,18 @@ import type { ReportEventMoreProps } from './ReportEventMore';export function ReportEvents() {const previous = useSelector((state) => state.report.previous);const selectedEvents = useSelector((state) => state.report.events);- const input = useSelector((state) => state.report);+ const startDate = useSelector((state) => state.report.startDate);+ const endDate = useSelector((state) => state.report.endDate);+ const range = useSelector((state) => state.report.range);+ const interval = useSelector((state) => state.report.interval);const dispatch = useDispatch();const { projectId } = useAppParams();- const eventNames = useEventNames(projectId, {- startDate: input.startDate,- endDate: input.endDate,- range: input.range,+ const eventNames = useEventNames({+ projectId,+ startDate,+ endDate,+ range,+ interval,});const dispatchChangeEvent = useDebounceFn((event: IChartEvent) => {diff --git a/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx b/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsxindex 5e228f805..bfa7de03b 100644--- a/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx+++ b/apps/dashboard/src/components/report/sidebar/filters/FilterItem.tsx@@ -26,7 +26,9 @@ interface FilterProps {export function FilterItem({ filter, event }: FilterProps) {const { projectId } = useAppParams();- const { range, startDate, endDate } = useSelector((state) => state.report);+ const { range, startDate, endDate, interval } = useSelector(+ (state) => state.report+ );const getLabel = useMappings();const dispatch = useDispatch();const potentialValues = api.chart.values.useQuery({@@ -34,6 +36,7 @@ export function FilterItem({ filter, event }: FilterProps) {property: filter.name,projectId,range,+ interval,startDate,endDate,});diff --git a/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx b/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsxindex 0cdbc7421..ba7c62c83 100644--- a/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx+++ b/apps/dashboard/src/components/report/sidebar/filters/FiltersCombobox.tsx@@ -15,7 +15,10 @@ interface FiltersComboboxProps {export function FiltersCombobox({ event }: FiltersComboboxProps) {const dispatch = useDispatch();- const { range, startDate, endDate } = useSelector((state) => state.report);+ const interval = useSelector((state) => state.report.interval);+ const range = useSelector((state) => state.report.range);+ const startDate = useSelector((state) => state.report.startDate);+ const endDate = useSelector((state) => state.report.endDate);const { projectId } = useAppParams();const query = api.chart.properties.useQuery(@@ -23,6 +26,7 @@ export function FiltersCombobox({ event }: FiltersComboboxProps) {event: event.name,projectId,range,+ interval,startDate,endDate,},diff --git a/apps/dashboard/src/hooks/useEventNames.ts b/apps/dashboard/src/hooks/useEventNames.tsindex 17d908d6f..d224f8866 100644--- a/apps/dashboard/src/hooks/useEventNames.ts+++ b/apps/dashboard/src/hooks/useEventNames.ts@@ -1,10 +1,8 @@import { api } from '@/trpc/client';-export function useEventNames(projectId: string, options?: any) {- const query = api.chart.events.useQuery({- projectId: projectId,- ...(options ? options : {}),- });-+export function useEventNames(+ params: Parameters<typeof api.chart.events.useQuery>[0]+) {+ const query = api.chart.events.useQuery(params);return query.data ?? [];}diff --git a/apps/dashboard/src/hooks/useEventProperties.ts b/apps/dashboard/src/hooks/useEventProperties.tsindex 732a1fcaf..bad5801ba 100644--- a/apps/dashboard/src/hooks/useEventProperties.ts+++ b/apps/dashboard/src/hooks/useEventProperties.ts@@ -1,10 +1,9 @@import { api } from '@/trpc/client';-export function useEventProperties(projectId: string, event?: string) {- const query = api.chart.properties.useQuery({- projectId: projectId,- event,- });+export function useEventProperties(+ params: Parameters<typeof api.chart.properties.useQuery>[0]+) {+ const query = api.chart.properties.useQuery(params);return query.data ?? [];}diff --git a/apps/dashboard/src/hooks/useEventValues.ts b/apps/dashboard/src/hooks/useEventValues.tsindex 6eda81be4..e54dbf0cb 100644--- a/apps/dashboard/src/hooks/useEventValues.ts+++ b/apps/dashboard/src/hooks/useEventValues.ts@@ -1,15 +1,8 @@import { api } from '@/trpc/client';export function useEventValues(- projectId: string,- event: string,- property: string+ params: Parameters<typeof api.chart.values.useQuery>[0]) {- const query = api.chart.values.useQuery({- projectId: projectId,- event,- property,- });-+ const query = api.chart.values.useQuery(params);return query.data?.values ?? [];}diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 9ea6128d1..62968f922 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -1,5 +1,8 @@import type { ResponseJSON } from '@clickhouse/client';import { createClient } from '@clickhouse/client';+import { escape } from 'sqlstring';++import type { IInterval } from '@openpanel/validation';export const TABLE_NAMES = {events: 'events_v2',@@ -126,6 +129,22 @@ export function formatClickhouseDate(return date.toISOString().replace('T', ' ').replace(/Z+$/, '');}+export function toDate(str: string, interval?: IInterval) {+ if (!interval || interval === 'minute' || interval === 'hour') {+ if (str.match(/\d{4}-\d{2}-\d{2}/)) {+ return escape(str);+ }++ return str;+ }++ if (str.match(/\d{4}-\d{2}-\d{2}/)) {+ return `toDate(${escape(str)})`;+ }++ return `toDate(${str})`;+}+export function convertClickhouseDateToJs(date: string) {return new Date(date.replace(' ', 'T') + 'Z');}diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex 25394f598..3c6eb27b1 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -6,7 +6,11 @@ import type {IGetChartDataInput,} from '@openpanel/validation';-import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse-client';+import {+ formatClickhouseDate,+ TABLE_NAMES,+ toDate,+} from '../clickhouse-client';import { createSqlBuilder } from '../sql-builder';function getPropertyKey(property: string) {@@ -67,21 +71,11 @@ export function getChartSql({sb.groupBy.date = 'date';if (startDate) {- sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;- // if (interval === 'minute' || interval === 'hour') {- // sb.where.startDate = `created_at >= '${formatClickhouseDate(startDate)}'`;- // } else {- // sb.where.startDate = `toDate(created_at) >= '${formatClickhouseDate(startDate, true)}'`;- // }+ sb.where.startDate = `${toDate('created_at', interval)} >= ${toDate(formatClickhouseDate(startDate), interval)}`;}if (endDate) {- sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;- // if (interval === 'minute' || interval === 'hour') {- // sb.where.endDate = `created_at <= '${formatClickhouseDate(endDate)}'`;- // } else {- // sb.where.endDate = `toDate(created_at) <= '${formatClickhouseDate(endDate, true)}'`;- // }+ sb.where.endDate = `${toDate('created_at', interval)} <= ${toDate(formatClickhouseDate(endDate), interval)}`;}if (breakdowns.length > 0 && limit) {diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 4b3a3a9bd..967074afe 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -1,22 +1,16 @@-import { subMonths } from 'date-fns';import { flatten, map, pipe, prop, sort, uniq } from 'ramda';import { escape } from 'sqlstring';import { z } from 'zod';-import { average, max, min, round, slug, sum } from '@openpanel/common';import {chQuery,createSqlBuilder,db,formatClickhouseDate,TABLE_NAMES,+ toDate,} from '@openpanel/db';-import { zChartInput, zRange } from '@openpanel/validation';-import type {- FinalChart,- IChartInput,- PreviousValue,-} from '@openpanel/validation';+import { zChartInput, zRange, zTimeInterval } from '@openpanel/validation';import { getProjectAccessCached } from '../access';import { TRPCAccessError } from '../errors';@@ -34,7 +28,8 @@ export const chartRouter = createTRPCRouter({.input(z.object({projectId: z.string(),- range: zRange.default('30d'),+ range: zRange,+ interval: zTimeInterval,startDate: z.string().nullish(),endDate: z.string().nullish(),})@@ -42,7 +37,7 @@ export const chartRouter = createTRPCRouter({.query(async ({ input: { projectId, ...input } }) => {const { startDate, endDate } = getChartStartEndDate(input);const events = await chQuery<{ name: string }>(- `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`+ `SELECT DISTINCT name FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)} AND ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`);return [@@ -58,7 +53,8 @@ export const chartRouter = createTRPCRouter({z.object({event: z.string().optional(),projectId: z.string(),- range: zRange.default('30d'),+ range: zRange,+ interval: zTimeInterval,startDate: z.string().nullish(),endDate: z.string().nullish(),})@@ -69,7 +65,7 @@ export const chartRouter = createTRPCRouter({`SELECT distinct mapKeys(properties) as keys from ${TABLE_NAMES.events} where ${event && event !== '*' ? `name = ${escape(event)} AND ` : ''} project_id = ${escape(projectId)} AND- created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}');`+ ${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`);const properties = events@@ -111,7 +107,8 @@ export const chartRouter = createTRPCRouter({event: z.string(),property: z.string(),projectId: z.string(),- range: zRange.default('30d'),+ range: zRange,+ interval: zTimeInterval,startDate: z.string().nullish(),endDate: z.string().nullish(),})@@ -137,7 +134,7 @@ export const chartRouter = createTRPCRouter({sb.select.values = `distinct ${property} as values`;}- sb.where.date = `created_at BETWEEN toDate('${formatClickhouseDate(startDate)}') AND toDate('${formatClickhouseDate(endDate)}')`;+ sb.where.date = `${toDate('created_at', input.interval)} BETWEEN ${toDate(formatClickhouseDate(startDate), input.interval)} AND ${toDate(formatClickhouseDate(endDate), input.interval)};`;const events = await chQuery<{ values: string[] }>(getSql());