OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: default last/first seen broken when clickhouse defaults to 1970
Carl-Gerhard Lindesvärd8c377c2066a3
diff --git a/apps/start/src/components/overview/overview-metric-card.tsx b/apps/start/src/components/overview/overview-metric-card.tsxindex 66cbdfd7f..a2ea8310f 100644--- a/apps/start/src/components/overview/overview-metric-card.tsx+++ b/apps/start/src/components/overview/overview-metric-card.tsx@@ -1,23 +1,16 @@-import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter';-import { cn } from '@/utils/cn';-import AutoSizer from 'react-virtualized-auto-sizer';-import { Area, AreaChart, Bar, BarChart, Tooltip } from 'recharts';--import { formatDate, timeAgo } from '@/utils/date';-import { getChartColor } from '@/utils/theme';import { getPreviousMetric } from '@openpanel/common';import { useEffect, useRef, useState } from 'react';+import AutoSizer from 'react-virtualized-auto-sizer';+import { Bar, BarChart, Tooltip } from 'recharts';import {- ChartTooltipContainer,- ChartTooltipHeader,- ChartTooltipItem,-} from '../charts/chart-tooltip';-import {- PreviousDiffIndicatorPure,getDiffIndicator,+ PreviousDiffIndicatorPure,} from '../report-chart/common/previous-diff-indicator';import { Skeleton } from '../skeleton';import { Tooltiper } from '../ui/tooltip';+import { fancyMinutes, useNumber } from '@/hooks/use-numer-formatter';+import { cn } from '@/utils/cn';+import { formatDate, timeAgo } from '@/utils/date';interface MetricCardProps {id: string;@@ -78,6 +71,9 @@ export function OverviewMetricCard({}if (unit === 'timeAgo') {+ if (!value) {+ return <>{'N/A'}</>;+ }return <>{timeAgo(new Date(value))}</>;}@@ -103,7 +99,7 @@ export function OverviewMetricCard({getPreviousMetric(current, previous)?.state,'#6ee7b7', // green'#fda4af', // red- '#93c5fd', // blue+ '#93c5fd' // blue);const renderTooltip = () => {@@ -115,7 +111,7 @@ export function OverviewMetricCard({{renderValue(data[currentIndex].current,'ml-1 font-light text-xl',- false,+ false)}</span></span>@@ -132,60 +128,60 @@ export function OverviewMetricCard({);};return (- <Tooltiper content={renderTooltip()} asChild sideOffset={-20}>+ <Tooltiper asChild content={renderTooltip()} sideOffset={-20}><button- type="button"className={cn('col-span-2 flex-1 shadow-[0_0_0_0.5px] shadow-border md:col-span-1',- active && 'bg-def-100',+ active && 'bg-def-100')}onClick={onClick}+ type="button"><div className={cn('group relative p-4')}><divclassName={cn(- 'absolute left-4 right-4 bottom-0 z-0 opacity-50 transition-opacity duration-300 group-hover:opacity-100',+ 'absolute right-4 bottom-0 left-4 z-0 opacity-50 transition-opacity duration-300 group-hover:opacity-100')}><AutoSizer style={{ height: 20 }}>{({ width }) => (<BarChart- width={width}- height={20}data={data}- style={{- background: 'transparent',- }}+ height={20}+ margin={{ top: 0, right: 0, left: 0, bottom: 0 }}onMouseMove={(event) => {setCurrentIndex(event.activeTooltipIndex ?? null);}}- margin={{ top: 0, right: 0, left: 0, bottom: 0 }}+ style={{+ background: 'transparent',+ }}+ width={width}><Tooltip content={() => null} cursor={false} /><BardataKey={'current'}- type="step"fill={graphColors}fillOpacity={1}- strokeWidth={0}isAnimationActive={false}+ strokeWidth={0}+ type="step"/></BarChart>)}</AutoSizer></div><OverviewMetricCardNumber- label={label}- value={renderValue(current, 'ml-1 font-light text-xl')}enhancer={<PreviousDiffIndicatorPureclassName="text-sm"- size="sm"inverted={inverted}+ size="sm"{...getPreviousMetric(current, previous)}/>}isLoading={isLoading}+ label={label}+ value={renderValue(current, 'ml-1 font-light text-xl')}/></div></button>@@ -207,9 +203,9 @@ export function OverviewMetricCardNumber({isLoading?: boolean;}) {return (- <div className={cn('min-w-0 col gap-2', className)}>+ <div className={cn('col min-w-0 gap-2', className)}><div className="flex min-w-0 items-center gap-2 text-left">- <span className="truncate text-sm font-medium text-muted-foreground leading-[1.1]">+ <span className="truncate font-medium text-muted-foreground text-sm leading-[1.1]">{label}</span></div>@@ -219,11 +215,11 @@ export function OverviewMetricCardNumber({<Skeleton className="h-6 w-12" /></div>) : (- <div className="truncate font-mono text-3xl leading-[1.1] font-bold w-full text-left">+ <div className="w-full truncate text-left font-bold font-mono text-3xl leading-[1.1]">{value}</div>)}- <div className="absolute right-0 top-0 bottom-0 center justify-center col pr-4">+ <div className="center col absolute top-0 right-0 bottom-0 justify-center pr-4">{enhancer}</div></div>diff --git a/apps/start/src/components/profiles/profile-metrics.tsx b/apps/start/src/components/profiles/profile-metrics.tsxindex eb30c53c2..6558c4ee6 100644--- a/apps/start/src/components/profiles/profile-metrics.tsx+++ b/apps/start/src/components/profiles/profile-metrics.tsx@@ -1,6 +1,5 @@-import { OverviewMetricCard } from '@/components/overview/overview-metric-card';-import type { IProfileMetrics } from '@openpanel/db';+import { OverviewMetricCard } from '@/components/overview/overview-metric-card';type Props = {data: IProfileMetrics;@@ -102,7 +101,7 @@ const PROFILE_METRICS = [export const ProfileMetrics = ({ data }: Props) => {return (- <div className="relative col-span-6 -m-4 mb-0 mt-0 md:m-0">+ <div className="relative col-span-6 -m-4 mt-0 mb-0 md:m-0"><div className="card grid grid-cols-2 overflow-hidden rounded-md md:grid-cols-4 lg:grid-cols-6">{PROFILE_METRICS.filter((metric) => {if (metric.hideOnZero && data[metric.key] === 0) {@@ -111,20 +110,20 @@ export const ProfileMetrics = ({ data }: Props) => {return true;}).map((metric) => (<OverviewMetricCard- key={metric.key}+ data={[]}id={metric.key}+ inverted={metric.inverted}+ isLoading={false}+ key={metric.key}label={metric.title}metric={{current:- metric.unit === 'timeAgo'- ? new Date(data[metric.key]).getTime()+ metric.unit === 'timeAgo' && data[metric.key]+ ? new Date(data[metric.key]!).getTime(): (data[metric.key] as number) || 0,previous: null,}}unit={metric.unit}- data={[]}- inverted={metric.inverted}- isLoading={false}/>))}</div>diff --git a/packages/common/src/date.ts b/packages/common/src/date.tsindex a6c320653..6a69d9cc5 100644--- a/packages/common/src/date.ts+++ b/packages/common/src/date.ts@@ -1,6 +1,6 @@-import { DateTime } from 'luxon';+export { DateTime } from 'luxon';-export { DateTime };+export type { DateTime };export function getTime(date: string | number | Date) {return new Date(date).getTime();diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex c1d306a2d..d2899f825 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -290,3 +290,8 @@ export function toDate(str: string, interval?: IInterval) {export function convertClickhouseDateToJs(date: string) {return new Date(`${date.replace(' ', 'T')}Z`);}++const ROLLUP_DATE_PREFIX = '1970-01-01';+export function isClickhouseDefaultMinDate(date: string): boolean {+ return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');+}diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.tsindex c0619a2b4..17b8d900d 100644--- a/packages/db/src/services/overview.service.ts+++ b/packages/db/src/services/overview.service.ts@@ -1,14 +1,13 @@import { average, sum } from '@openpanel/common';import { chartColors } from '@openpanel/constants';-import { getCache } from '@openpanel/redis';import { type IChartEventFilter, zTimeInterval } from '@openpanel/validation';-import { omit } from 'ramda';import sqlstring from 'sqlstring';import { z } from 'zod';import {- TABLE_NAMES,ch,convertClickhouseDateToJs,+ isClickhouseDefaultMinDate,+ TABLE_NAMES,} from '../clickhouse/client';import { clix } from '../clickhouse/query-builder';import {@@ -172,24 +171,12 @@ export type IGetMapDataInput = z.infer<typeof zGetMapDataInput> & {export class OverviewService {constructor(private client: typeof ch) {}- // Helper methods- private isRollupRow(date: string): boolean {- // The rollup row has date 1970-01-01 00:00:00 (epoch) from ClickHouse.- // After transform with `new Date().toISOString()`, this becomes an ISO string.- // Due to timezone handling in JavaScript's Date constructor (which interprets- // the input as local time), the UTC date might become:- // - 1969-12-31T... for positive UTC offsets (e.g., UTC+8)- // - 1970-01-01T... for UTC or negative offsets- // We check for both year prefixes to handle all server timezones.- return date.startsWith(ROLLUP_DATE_PREFIX) || date.startsWith('1969-12-31');- }-private getFillConfig(interval: string, startDate: string, endDate: string) {const useDateOnly = ['month', 'week'].includes(interval);return {from: clix.toStartOf(clix.datetime(startDate, useDateOnly ? 'toDate' : 'toDateTime'),- interval as any,+ interval as any),to: clix.datetime(endDate, useDateOnly ? 'toDate' : 'toDateTime'),step: clix.toInterval('1', interval as any),@@ -234,12 +221,12 @@ export class OverviewService {private mergeRevenueIntoSeries<T extends { date: string }>(series: T[],- revenueData: { date: string; total_revenue: number }[],+ revenueData: { date: string; total_revenue: number }[]): (T & { total_revenue: number })[] {const revenueByDate = new Map(revenueData- .filter((r) => !this.isRollupRow(r.date))- .map((r) => [r.date, r.total_revenue]),+ .filter((r) => !isClickhouseDefaultMinDate(r.date))+ .map((r) => [r.date, r.total_revenue]));return series.map((row) => ({...row,@@ -248,10 +235,11 @@ export class OverviewService {}private getOverallRevenue(- revenueData: { date: string; total_revenue: number }[],+ revenueData: { date: string; total_revenue: number }[]): number {return (- revenueData.find((r) => this.isRollupRow(r.date))?.total_revenue ?? 0+ revenueData.find((r) => isClickhouseDefaultMinDate(r.date))+ ?.total_revenue ?? 0);}@@ -263,7 +251,7 @@ export class OverviewService {startDate: string;endDate: string;timezone: string;- },+ }): ReturnType<typeof clix> {if (!this.isPageFilter(params.filters)) {query.rawWhere(this.getRawWhereClause('sessions', params.filters));@@ -276,7 +264,7 @@ export class OverviewService {.where('id','IN',- clix.exp('(SELECT session_id FROM distinct_sessions)'),+ clix.exp('(SELECT session_id FROM distinct_sessions)'));}@@ -475,14 +463,14 @@ export class OverviewService {clix(this.client, timezone).select(['bounce_rate']).from('session_agg')- .where('date', '=', rollupDate),+ .where('date', '=', rollupDate)).with('daily_session_stats',clix(this.client, timezone).select(['date', 'bounce_rate']).from('session_agg')- .where('date', '!=', rollupDate),+ .where('date', '!=', rollupDate)).with('overall_unique_visitors', overallUniqueVisitorsQuery).select<{@@ -512,7 +500,7 @@ export class OverviewService {.from(`${TABLE_NAMES.events} AS e`).leftJoin('daily_session_stats AS dss',- `${clix.toStartOf('e.created_at', interval as any)} = dss.date`,+ `${clix.toStartOf('e.created_at', interval as any)} = dss.date`).where('e.project_id', '=', projectId).where('e.name', '=', 'screen_view')@@ -551,7 +539,7 @@ export class OverviewService {(item) =>item.overall_bounce_rate !== null ||item.overall_total_sessions !== null ||- item.overall_unique_visitors !== null,+ item.overall_unique_visitors !== null);return {@@ -560,11 +548,11 @@ export class OverviewService {unique_visitors: anyRowWithData?.overall_unique_visitors ?? 0,total_sessions: anyRowWithData?.overall_total_sessions ?? 0,avg_session_duration: average(- mainRes.map((item) => item.avg_session_duration),+ mainRes.map((item) => item.avg_session_duration)),total_screen_views: sum(mainRes.map((item) => item.total_screen_views)),views_per_session: average(- mainRes.map((item) => item.views_per_session),+ mainRes.map((item) => item.views_per_session)),total_revenue: overallRevenue,},@@ -591,7 +579,7 @@ export class OverviewService {return item;}return item;- }),+ }));return Object.values(where).join(' AND ');@@ -879,7 +867,7 @@ export class OverviewService {startDate,endDate,timezone,- },+ });const timeSeriesData = await mainTimeSeriesQuery.execute();@@ -1066,7 +1054,7 @@ export class OverviewService {.from('paths_deduped_cte').having('length(paths)', '>=', 2)// ONLY sessions starting with top entry pages- .having('paths[1]', 'IN', topEntryPages),+ .having('paths[1]', 'IN', topEntryPages)).select<{source: string;@@ -1081,8 +1069,8 @@ export class OverviewService {]).from(clix.exp(- '(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)',- ),+ '(SELECT arrayJoin(arrayMap(i -> (paths[i], paths[i + 1], i), range(1, length(paths)))) as pair FROM session_paths WHERE length(paths) >= 2)'+ )).groupBy(['source', 'target', 'step']).orderBy('step', 'ASC')@@ -1143,7 +1131,9 @@ export class OverviewService {for (const t of fromSource) {// Skip self-loops- if (t.source === t.target) continue;+ if (t.source === t.target) {+ continue;+ }const targetNodeId = getNodeId(t.target, step + 1);@@ -1180,7 +1170,9 @@ export class OverviewService {}// Stop if no more nodes to process- if (activeNodes.size === 0) break;+ if (activeNodes.size === 0) {+ break;+ }}// Step 5: Filter links by threshold (0.25% of total sessions)@@ -1235,22 +1227,24 @@ export class OverviewService {}).sort((a, b) => {// Sort by step first, then by value descending- if (a.step !== b.step) return a.step - b.step;+ if (a.step !== b.step) {+ return a.step - b.step;+ }return b.value - a.value;});// Sanity check: Ensure all link endpoints exist in nodesconst nodeIds = new Set(finalNodes.map((n) => n.id));const invalidLinks = filteredLinks.filter(- (link) => !nodeIds.has(link.source) || !nodeIds.has(link.target),+ (link) => !(nodeIds.has(link.source) && nodeIds.has(link.target)));if (invalidLinks.length > 0) {console.warn(- `UserJourney: Found ${invalidLinks.length} links with missing nodes`,+ `UserJourney: Found ${invalidLinks.length} links with missing nodes`);// Remove invalid linksconst validLinks = filteredLinks.filter(- (link) => nodeIds.has(link.source) && nodeIds.has(link.target),+ (link) => nodeIds.has(link.source) && nodeIds.has(link.target));return {nodes: finalNodes,@@ -1260,7 +1254,9 @@ export class OverviewService {// Sanity check: Ensure steps are monotonic (should always be true, but verify)const stepsValid = finalNodes.every((node, idx, arr) => {- if (idx === 0) return true;+ if (idx === 0) {+ return true;+ }return node.step! >= arr[idx - 1]!.step!;});if (!stepsValid) {diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 41358791c..424382b45 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -1,23 +1,21 @@-import { omit, uniq } from 'ramda';-import sqlstring from 'sqlstring';-import { strip, toObject } from '@openpanel/common';import { cacheable } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';-+import { uniq } from 'ramda';+import sqlstring from 'sqlstring';import { profileBuffer } from '../buffers';import {- TABLE_NAMES,- ch,chQuery,convertClickhouseDateToJs,formatClickhouseDate,+ isClickhouseDefaultMinDate,+ TABLE_NAMES,} from '../clickhouse/client';import { createSqlBuilder } from '../sql-builder';-export type IProfileMetrics = {- lastSeen: Date;- firstSeen: Date;+export interface IProfileMetrics {+ lastSeen: Date | null;+ firstSeen: Date | null;screenViews: number;sessions: number;durationAvg: number;@@ -29,7 +27,7 @@ export type IProfileMetrics = {conversionEvents: number;avgTimeBetweenSessions: number;revenue: number;-};+}export function getProfileMetrics(profileId: string, projectId: string) {return chQuery<Omit<IProfileMetrics, 'lastSeen' | 'firstSeen'> & {@@ -100,8 +98,12 @@ export function getProfileMetrics(profileId: string, projectId: string) {.then((data) => {return {...data,- lastSeen: convertClickhouseDateToJs(data.lastSeen),- firstSeen: convertClickhouseDateToJs(data.firstSeen),+ lastSeen: isClickhouseDefaultMinDate(data.lastSeen)+ ? null+ : convertClickhouseDateToJs(data.lastSeen),+ firstSeen: isClickhouseDefaultMinDate(data.firstSeen)+ ? null+ : convertClickhouseDateToJs(data.firstSeen),};});}@@ -127,7 +129,7 @@ export async function getProfileById(id: string, projectId: string) {last_value(is_external) as is_external,last_value(properties) as properties,last_value(created_at) as created_at- FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`,+ FROM ${TABLE_NAMES.profiles} FINAL WHERE id = ${sqlstring.escape(String(id))} AND project_id = ${sqlstring.escape(projectId)} GROUP BY id, project_id ORDER BY created_at DESC LIMIT 1`);if (!profile) {@@ -169,7 +171,7 @@ export async function getProfiles(ids: string[], projectId: string) {project_id = ${sqlstring.escape(projectId)} ANDid IN (${filteredIds.map((id) => sqlstring.escape(id)).join(',')})GROUP BY id, project_id- `,+ `);return data.map(transformProfile);@@ -221,7 +223,7 @@ export async function getProfileListCount({return data[0]?.count ?? 0;}-export type IServiceProfile = {+export interface IServiceProfile {id: string;email: string;avatar: string;@@ -245,7 +247,7 @@ export type IServiceProfile = {model?: string;referrer?: string;};-};+}export interface IClickhouseProfile {id: string;@@ -289,7 +291,7 @@ export function transformProfile({};}-export async function upsertProfile(+export function upsertProfile({id,firstName,@@ -300,7 +302,7 @@ export async function upsertProfile(projectId,isExternal,}: IServiceUpsertProfile,- isFromEvent = false,+ isFromEvent = false) {const profile: IClickhouseProfile = {id,