Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix:buffers

Carl-Gerhard Lindesvärda1ce71ffb660
diff --git a/apps/api/src/controllers/live.controller.ts b/apps/api/src/controllers/live.controller.ts
index 488e67138..d73f267ec 100644
--- a/apps/api/src/controllers/live.controller.ts
+++ b/apps/api/src/controllers/live.controller.ts
@@ -1,10 +1,7 @@
import type { WebSocket } from '@fastify/websocket';
import { eventBuffer } from '@openpanel/db';
import { setSuperJson } from '@openpanel/json';
-import {
- psubscribeToPublishedEvent,
- subscribeToPublishedEvent,
-} from '@openpanel/redis';
+import { subscribeToPublishedEvent } from '@openpanel/redis';
import { getProjectAccess } from '@openpanel/trpc';
import { getOrganizationAccess } from '@openpanel/trpc/src/access';
import type { FastifyRequest } from 'fastify';
@@ -39,19 +36,8 @@ export function wsVisitors(
}
);
- const punsubscribe = psubscribeToPublishedEvent(
- '__keyevent@0__:expired',
- (key) => {
- const [, , projectId] = key.split(':');
- if (projectId === params.projectId) {
- sendCount();
- }
- }
- );
-
socket.on('close', () => {
unsubscribe();
- punsubscribe();
});
}
diff --git a/apps/start/src/components/overview/live-counter.tsx b/apps/start/src/components/overview/live-counter.tsx
index 2333f5c09..c3d5e7336 100644
--- a/apps/start/src/components/overview/live-counter.tsx
+++ b/apps/start/src/components/overview/live-counter.tsx
@@ -1,61 +1,25 @@
-import { TooltipComplete } from '@/components/tooltip-complete';
-import { useDebounceState } from '@/hooks/use-debounce-state';
-import useWS from '@/hooks/use-ws';
-import { useTRPC } from '@/integrations/trpc/react';
-import { cn } from '@/utils/cn';
-import { useQuery, useQueryClient } from '@tanstack/react-query';
-import { useEffect, useRef } from 'react';
+import { useQueryClient } from '@tanstack/react-query';
+import { useCallback } from 'react';
import { toast } from 'sonner';
import { AnimatedNumber } from '../animated-number';
+import { TooltipComplete } from '@/components/tooltip-complete';
+import { useLiveCounter } from '@/hooks/use-live-counter';
+import { cn } from '@/utils/cn';
export interface LiveCounterProps {
projectId: string;
shareId?: string;
}
-const FIFTEEN_SECONDS = 1000 * 30;
-
export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
- const trpc = useTRPC();
const client = useQueryClient();
- const counter = useDebounceState(0, 1000);
- const lastRefresh = useRef(Date.now());
- const query = useQuery(
- trpc.overview.liveVisitors.queryOptions({
- projectId,
- shareId,
- }),
- );
-
- useEffect(() => {
- if (query.data) {
- counter.set(query.data);
- }
- }, [query.data]);
-
- useWS<number>(
- `/live/visitors/${projectId}`,
- (value) => {
- if (!Number.isNaN(value)) {
- counter.set(value);
- if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
- lastRefresh.current = Date.now();
- if (!document.hidden) {
- toast('Refreshed data');
- client.refetchQueries({
- type: 'active',
- });
- }
- }
- }
- },
- {
- debounce: {
- delay: 1000,
- maxWait: 5000,
- },
- },
- );
+ const onRefresh = useCallback(() => {
+ toast('Refreshed data');
+ client.refetchQueries({
+ type: 'active',
+ });
+ }, [client]);
+ const counter = useLiveCounter({ projectId, shareId, onRefresh });
return (
<TooltipComplete
@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {
<div
className={cn(
'h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',
- counter.debounced === 0 && 'bg-destructive opacity-0',
+ counter.debounced === 0 && 'bg-destructive opacity-0'
)}
/>
<div
className={cn(
- 'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
- counter.debounced === 0 && 'bg-destructive',
+ 'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',
+ counter.debounced === 0 && 'bg-destructive'
)}
/>
</div>
diff --git a/apps/start/src/hooks/use-live-counter.ts b/apps/start/src/hooks/use-live-counter.ts
new file mode 100644
index 000000000..d8e0d7eda
--- /dev/null
+++ b/apps/start/src/hooks/use-live-counter.ts
@@ -0,0 +1,81 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query';
+import { useEffect, useRef } from 'react';
+import { useDebounceState } from './use-debounce-state';
+import useWS from './use-ws';
+import { useTRPC } from '@/integrations/trpc/react';
+
+const FIFTEEN_SECONDS = 1000 * 15;
+/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */
+const FALLBACK_STALE_MS = 1000 * 60;
+
+export function useLiveCounter({
+ projectId,
+ shareId,
+ onRefresh,
+}: {
+ projectId: string;
+ shareId?: string;
+ onRefresh?: () => void;
+}) {
+ const trpc = useTRPC();
+ const queryClient = useQueryClient();
+ const counter = useDebounceState(0, 1000);
+ const lastRefresh = useRef(Date.now());
+ const query = useQuery(
+ trpc.overview.liveVisitors.queryOptions({
+ projectId,
+ shareId: shareId ?? undefined,
+ })
+ );
+
+ useEffect(() => {
+ if (query.data) {
+ counter.set(query.data);
+ }
+ }, [query.data]);
+
+ useWS<number>(
+ `/live/visitors/${projectId}`,
+ (value) => {
+ if (!Number.isNaN(value)) {
+ counter.set(value);
+ if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {
+ lastRefresh.current = Date.now();
+ if (!document.hidden) {
+ onRefresh?.();
+ }
+ }
+ }
+ },
+ {
+ debounce: {
+ delay: 1000,
+ maxWait: 5000,
+ },
+ }
+ );
+
+ useEffect(() => {
+ const id = setInterval(async () => {
+ if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {
+ return;
+ }
+ const data = await queryClient.fetchQuery(
+ trpc.overview.liveVisitors.queryOptions(
+ {
+ projectId,
+ shareId: shareId ?? undefined,
+ },
+ // Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.
+ { staleTime: 0 }
+ )
+ );
+ counter.set(data);
+ lastRefresh.current = Date.now();
+ }, FALLBACK_STALE_MS);
+
+ return () => clearInterval(id);
+ }, [projectId, shareId, trpc, queryClient, counter.set]);
+
+ return counter;
+}
diff --git a/apps/start/src/routes/widget/counter.tsx b/apps/start/src/routes/widget/counter.tsx
index a55e7c81a..d745aa62e 100644
--- a/apps/start/src/routes/widget/counter.tsx
+++ b/apps/start/src/routes/widget/counter.tsx
@@ -1,12 +1,11 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
+import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import { Ping } from '@/components/ping';
-import { useNumber } from '@/hooks/use-numer-formatter';
import useWS from '@/hooks/use-ws';
import { useTRPC } from '@/integrations/trpc/react';
import type { RouterOutputs } from '@/trpc/client';
-import { useQuery, useQueryClient } from '@tanstack/react-query';
-import { createFileRoute } from '@tanstack/react-router';
-import { z } from 'zod';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({
});
function RouteComponent() {
- const { shareId, limit, color } = Route.useSearch();
+ const { shareId } = Route.useSearch();
const trpc = useTRPC();
// Fetch widget data
const { data, isLoading } = useQuery(
- trpc.widget.counter.queryOptions({ shareId }),
+ trpc.widget.counter.queryOptions({ shareId })
);
if (isLoading) {
return (
- <div className="flex items-center gap-2 px-2 h-8">
+ <div className="flex h-8 items-center gap-2 px-2">
<Ping />
- <AnimatedNumber value={0} suffix=" unique visitors" />
+ <AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
if (!data) {
return (
- <div className="flex items-center gap-2 px-2 h-8">
+ <div className="flex h-8 items-center gap-2 px-2">
<Ping className="bg-orange-500" />
- <AnimatedNumber value={0} suffix=" unique visitors" />
+ <AnimatedNumber suffix=" unique visitors" value={0} />
</div>
);
}
- return <CounterWidget shareId={shareId} data={data} />;
+ return <CounterWidget data={data} shareId={shareId} />;
}
interface RealtimeWidgetProps {
@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {
function CounterWidget({ shareId, data }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
- const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
`/live/visitors/${data.projectId}`,
- (res) => {
+ () => {
if (!document.hidden) {
queryClient.refetchQueries(
- trpc.widget.counter.queryFilter({ shareId }),
+ trpc.widget.counter.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
- maxWait: 60000,
+ maxWait: 60_000,
},
- },
+ }
);
return (
- <div className="flex items-center gap-2 px-2 h-8">
+ <div className="flex h-8 items-center gap-2 px-2">
<Ping />
- <AnimatedNumber value={data.counter} suffix=" unique visitors" />
+ <AnimatedNumber suffix=" unique visitors" value={data.counter} />
</div>
);
}
diff --git a/apps/start/src/routes/widget/realtime.tsx b/apps/start/src/routes/widget/realtime.tsx
index a60a17a8e..8e1a4cdb3 100644
--- a/apps/start/src/routes/widget/realtime.tsx
+++ b/apps/start/src/routes/widget/realtime.tsx
@@ -1,3 +1,15 @@
+import { useQuery, useQueryClient } from '@tanstack/react-query';
+import { createFileRoute } from '@tanstack/react-router';
+import type React from 'react';
+import {
+ Bar,
+ BarChart,
+ ResponsiveContainer,
+ Tooltip,
+ XAxis,
+ YAxis,
+} from 'recharts';
+import { z } from 'zod';
import { AnimatedNumber } from '@/components/animated-number';
import {
ChartTooltipContainer,
@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';
import type { RouterOutputs } from '@/trpc/client';
import { cn } from '@/utils/cn';
import { getChartColor } from '@/utils/theme';
-import { useQuery, useQueryClient } from '@tanstack/react-query';
-import { createFileRoute } from '@tanstack/react-router';
-import type React from 'react';
-import {
- Bar,
- BarChart,
- ResponsiveContainer,
- Tooltip,
- XAxis,
- YAxis,
-} from 'recharts';
-import { z } from 'zod';
const widgetSearchSchema = z.object({
shareId: z.string(),
@@ -44,7 +44,7 @@ function RouteComponent() {
// Fetch widget data
const { data: widgetData, isLoading } = useQuery(
- trpc.widget.realtimeData.queryOptions({ shareId }),
+ trpc.widget.realtimeData.queryOptions({ shareId })
);
if (isLoading) {
@@ -53,10 +53,10 @@ function RouteComponent() {
if (!widgetData) {
return (
- <div className="flex h-screen w-full center-center bg-background text-foreground col p-4">
- <LogoSquare className="size-10 mb-4" />
- <h1 className="text-xl font-semibold">Widget not found</h1>
- <p className="mt-2 text-sm text-muted-foreground">
+ <div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">
+ <LogoSquare className="mb-4 size-10" />
+ <h1 className="font-semibold text-xl">Widget not found</h1>
+ <p className="mt-2 text-muted-foreground text-sm">
This widget is not available or has been removed.
</p>
</div>
@@ -65,10 +65,10 @@ function RouteComponent() {
return (
<RealtimeWidget
- shareId={shareId}
- limit={limit}
- data={widgetData}
color={color}
+ data={widgetData}
+ limit={limit}
+ shareId={shareId}
/>
);
}
@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {
function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
- const number = useNumber();
// WebSocket subscription for real-time updates
useWS<number>(
@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
() => {
if (!document.hidden) {
queryClient.refetchQueries(
- trpc.widget.realtimeData.queryFilter({ shareId }),
+ trpc.widget.realtimeData.queryFilter({ shareId })
);
}
},
{
debounce: {
delay: 1000,
- maxWait: 60000,
+ maxWait: 60_000,
},
- },
+ }
);
const maxDomain =
@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
const referrers = data.referrers.length > 0 ? 1 : 0;
const paths = data.paths.length > 0 ? 1 : 0;
const value = countries + referrers + paths;
- if (value === 3) return 'md:grid-cols-3';
- if (value === 2) return 'md:grid-cols-2';
+ if (value === 3) {
+ return 'md:grid-cols-3';
+ }
+ if (value === 2) {
+ return 'md:grid-cols-2';
+ }
return 'md:grid-cols-1';
})();
@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
<div className="flex h-screen w-full flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="p-6 pb-3">
- <div className="flex items-center justify-between w-full h-4">
- <div className="flex items-center gap-3 w-full">
+ <div className="flex h-4 w-full items-center justify-between">
+ <div className="flex w-full items-center gap-3">
<Ping />
- <div className="text-sm font-medium text-muted-foreground flex-1">
+ <div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
{data.project.domain && <SerieIcon name={data.project.domain} />}
@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
</div>
<div className="row">
- <div className="font-mono text-6xl font-bold h-18 text-foreground">
+ <div className="h-18 font-bold font-mono text-6xl text-foreground">
<AnimatedNumber value={data.liveCount} />
</div>
</div>
- <div className="flex h-20 w-full flex-col -mt-4">
+ <div className="-mt-4 flex h-20 w-full flex-col">
<div className="flex-1">
- <ResponsiveContainer width="100%" height="100%">
+ <ResponsiveContainer height="100%" width="100%">
<BarChart
data={data.histogram}
margin={{ top: 0, right: 0, left: 0, bottom: 0 }}
@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
cursor={{ fill: 'var(--def-100)', radius: 4 }}
/>
<XAxis
- dataKey="time"
axisLine={false}
- tickLine={false}
+ dataKey="time"
+ interval="preserveStartEnd"
tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}
+ tickLine={false}
ticks={[
data.histogram[0].time,
data.histogram[data.histogram.length - 1].time,
]}
- interval="preserveStartEnd"
/>
- <YAxis hide domain={[0, maxDomain]} />
+ <YAxis domain={[0, maxDomain]} hide />
<Bar
dataKey="sessionCount"
+ fill={color || 'var(--chart-0)'}
isAnimationActive={false}
radius={[4, 4, 4, 4]}
- fill={color || 'var(--chart-0)'}
/>
</BarChart>
</ResponsiveContainer>
@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{(data.countries.length > 0 ||
data.referrers.length > 0 ||
data.paths.length > 0) && (
- <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">
+ <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6">
<div className={cn('grid grid-cols-1 gap-6', grids)}>
{/* Countries */}
{data.countries.length > 0 && (
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.countries,
- limit,
+ limit
);
return (
<>
{visible.map((item) => (
- <RowItem key={item.country} count={item.count}>
+ <RowItem count={item.count} key={item.country}>
<div className="flex items-center gap-2">
<SerieIcon name={item.country} />
<span className="text-sm">
@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Referrers */}
{data.referrers.length > 0 && (
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.referrers,
- limit,
+ limit
);
return (
<>
{visible.map((item) => (
- <RowItem key={item.referrer} count={item.count}>
+ <RowItem count={item.count} key={item.referrer}>
<div className="flex items-center gap-2">
<SerieIcon name={item.referrer} />
<span className="truncate text-sm">
@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
{/* Paths */}
{data.paths.length > 0 && (
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
{(() => {
const { visible, rest, restCount } = getRestItems(
data.paths,
- limit,
+ limit
);
return (
<>
{visible.map((item) => (
- <RowItem key={item.path} count={item.count}>
+ <RowItem count={item.count} key={item.path}>
<span className="truncate text-sm">
{item.path}
</span>
@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {
}
// Custom tooltip component that uses portals to escape overflow hidden
-const CustomTooltip = ({ active, payload, coordinate }: any) => {
+const CustomTooltip = ({ active, payload }: any) => {
const number = useNumber();
- if (!active || !payload || !payload.length) {
+ if (!(active && payload && payload.length)) {
return null;
}
@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {
function RowItem({
children,
count,
-}: { children: React.ReactNode; count: number }) {
+}: {
+ children: React.ReactNode;
+ count: number;
+}) {
const number = useNumber();
return (
- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
{children}
<span className="font-semibold">{number.short(count)}</span>
</div>
@@ -340,7 +346,7 @@ function RowItem({
function getRestItems<T extends { count: number }>(
items: T[],
- limit: number,
+ limit: number
): { visible: T[]; rest: T[]; restCount: number } {
const visible = items.slice(0, limit);
const rest = items.slice(limit);
@@ -375,7 +381,7 @@ function RestRow({
: 'paths';
return (
- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">
+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">
<span className="truncate">
{firstName} and {otherCount} more {typeLabel}...
</span>
@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
const itemCount = Math.min(limit, 5);
return (
- <div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">
+ <div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">
{/* Header with live counter */}
<div className="border-b p-6 pb-3">
- <div className="flex items-center justify-between w-full h-4">
- <div className="flex items-center gap-3 w-full">
+ <div className="flex h-4 w-full items-center justify-between">
+ <div className="flex w-full items-center gap-3">
<div className="size-2 rounded-full bg-muted" />
- <div className="text-sm font-medium text-muted-foreground flex-1">
+ <div className="flex-1 font-medium text-muted-foreground text-sm">
USERS IN LAST 30 MINUTES
</div>
</div>
@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
</div>
<div className="row">
- <div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">
- <div className="h-full w-6 bg-muted rounded" />
- <div className="h-full w-6 bg-muted rounded" />
+ <div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">
+ <div className="h-full w-6 rounded bg-muted" />
+ <div className="h-full w-6 rounded bg-muted" />
</div>
</div>
- <div className="flex h-20 w-full flex-col -mt-4 pb-2.5">
- <div className="flex-1 row gap-1 h-full">
+ <div className="-mt-4 flex h-20 w-full flex-col pb-2.5">
+ <div className="row h-full flex-1 gap-1">
{SKELETON_HISTOGRAM.map((item, index) => (
<div
+ className="mt-auto h-full w-full rounded bg-muted"
key={index.toString()}
style={{ height: `${item}%` }}
- className="h-full w-full bg-muted rounded mt-auto"
/>
))}
</div>
<div className="row justify-between pt-2">
- <div className="h-3 w-8 bg-muted rounded" />
- <div className="h-3 w-8 bg-muted rounded" />
+ <div className="h-3 w-8 rounded bg-muted" />
+ <div className="h-3 w-8 rounded bg-muted" />
</div>
</div>
</div>
- <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">
+ <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">
{/* Countries, Referrers, and Paths skeleton */}
<div className="grid grid-cols-1 gap-6 md:grid-cols-3">
{/* Countries skeleton */}
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
COUNTRY
</div>
<div className="col">
@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Referrers skeleton */}
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
REFERRER
</div>
<div className="col">
@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
{/* Paths skeleton */}
<div className="flex flex-col">
- <div className="mb-3 text-xs font-medium text-muted-foreground">
+ <div className="mb-3 font-medium text-muted-foreground text-xs">
PATH
</div>
<div className="col">
@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {
function RowItemSkeleton() {
return (
- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">
+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm">
<div className="flex items-center gap-2">
<div className="size-5 rounded bg-muted" />
- <div className="h-4 w-24 bg-muted rounded" />
+ <div className="h-4 w-24 rounded bg-muted" />
</div>
- <div className="h-4 w-8 bg-muted rounded" />
+ <div className="h-4 w-8 rounded bg-muted" />
</div>
);
}
diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.ts
index 178f94545..e269ef0b1 100644
--- a/packages/db/src/buffers/event-buffer.test.ts
+++ b/packages/db/src/buffers/event-buffer.test.ts
@@ -1,6 +1,7 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { ch } from '../clickhouse/client';
+import * as chClient from '../clickhouse/client';
+const { ch } = chClient;
// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
vi.mock('../services/event.service', () => ({}));
@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';
const redis = getRedisCache();
beforeEach(async () => {
- await redis.flushdb();
+ const keys = await redis.keys('event*');
+ if (keys.length > 0) await redis.del(...keys);
});
afterAll(async () => {
@@ -209,18 +211,16 @@ describe('EventBuffer', () => {
});
it('tracks active visitors', async () => {
- const event = {
- project_id: 'p9',
- profile_id: 'u9',
- name: 'custom',
- created_at: new Date().toISOString(),
- } as any;
-
- eventBuffer.add(event);
- await eventBuffer.flush();
+ const querySpy = vi
+ .spyOn(chClient, 'chQuery')
+ .mockResolvedValueOnce([{ count: 2 }] as any);
const count = await eventBuffer.getActiveVisitorCount('p9');
- expect(count).toBeGreaterThanOrEqual(1);
+ expect(count).toBe(2);
+ expect(querySpy).toHaveBeenCalledOnce();
+ expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
+
+ querySpy.mockRestore();
});
it('handles multiple sessions independently — all events go to buffer', async () => {
@@ -273,4 +273,24 @@ describe('EventBuffer', () => {
expect(await eventBuffer.getBufferSize()).toBe(5);
});
+
+ it('retains events in queue when ClickHouse insert fails', async () => {
+ eventBuffer.add({
+ project_id: 'p12',
+ name: 'event1',
+ created_at: new Date().toISOString(),
+ } as any);
+ await eventBuffer.flush();
+
+ const insertSpy = vi
+ .spyOn(ch, 'insert')
+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+ await eventBuffer.processBuffer();
+
+ // Events must still be in the queue — not lost
+ expect(await eventBuffer.getBufferSize()).toBe(1);
+
+ insertSpy.mockRestore();
+ });
});
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index 2e5d43dce..883812adc 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -1,6 +1,6 @@
import { getSafeJson } from '@openpanel/json';
-import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';
-import { ch } from '../clickhouse/client';
+import { getRedisCache, publishEvent } from '@openpanel/redis';
+import { ch, chQuery } from '../clickhouse/client';
import type { IClickhouseEvent } from '../services/event.service';
import { BaseBuffer } from './base-buffer';
@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {
/** Tracks consecutive flush failures for observability; reset on success. */
private flushRetryCount = 0;
- private activeVisitorsExpiration = 60 * 5; // 5 minutes
- /** How often (ms) we refresh the heartbeat key + zadd per visitor. */
- private heartbeatRefreshMs = 60_000; // 1 minute
- private lastHeartbeat = new Map<string, number>();
private queueKey = 'event_buffer:queue';
protected bufferCounterKey = 'event_buffer:total_count';
@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {
for (const event of eventsToFlush) {
multi.rpush(this.queueKey, JSON.stringify(event));
- if (event.profile_id) {
- this.incrementActiveVisitorCount(
- multi,
- event.project_id,
- event.profile_id
- );
- }
}
multi.incrby(this.bufferCounterKey, eventsToFlush.length);
await multi.exec();
this.flushRetryCount = 0;
- this.pruneHeartbeatMap();
} catch (error) {
// Re-queue failed events at the front to preserve order and avoid data loss
this.pendingEvents = eventsToFlush.concat(this.pendingEvents);
@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {
}
}
- public async getBufferSize() {
+ public getBufferSize() {
return this.getBufferSizeWithCounter(async () => {
const redis = getRedisCache();
return await redis.llen(this.queueKey);
});
}
- private pruneHeartbeatMap() {
- const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
- for (const [key, ts] of this.lastHeartbeat) {
- if (ts < cutoff) {
- this.lastHeartbeat.delete(key);
- }
- }
- }
-
- private incrementActiveVisitorCount(
- multi: ReturnType<Redis['multi']>,
- projectId: string,
- profileId: string
- ) {
- const key = `${projectId}:${profileId}`;
- const now = Date.now();
- const last = this.lastHeartbeat.get(key) ?? 0;
-
- if (now - last < this.heartbeatRefreshMs) {
- return;
- }
-
- this.lastHeartbeat.set(key, now);
- const zsetKey = `live:visitors:${projectId}`;
- const heartbeatKey = `live:visitor:${projectId}:${profileId}`;
- multi
- .zadd(zsetKey, now, profileId)
- .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);
- }
-
public async getActiveVisitorCount(projectId: string): Promise<number> {
- const redis = getRedisCache();
- const zsetKey = `live:visitors:${projectId}`;
- const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;
-
- const multi = redis.multi();
- multi
- .zremrangebyscore(zsetKey, '-inf', cutoff)
- .zcount(zsetKey, cutoff, '+inf');
-
- const [, count] = (await multi.exec()) as [
- [Error | null, any],
- [Error | null, number],
- ];
-
- return count[1] || 0;
+ const rows = await chQuery<{ count: number }>(
+ `SELECT uniq(profile_id) AS count
+ FROM events
+ WHERE project_id = '${projectId}'
+ AND profile_id != ''
+ AND created_at >= now() - INTERVAL 5 MINUTE`
+ );
+ return rows[0]?.count ?? 0;
}
}
diff --git a/packages/db/src/buffers/profile-buffer.test.ts b/packages/db/src/buffers/profile-buffer.test.ts
index bc7d39d22..2dc757f97 100644
--- a/packages/db/src/buffers/profile-buffer.test.ts
+++ b/packages/db/src/buffers/profile-buffer.test.ts
@@ -1,6 +1,5 @@
import { getRedisCache } from '@openpanel/redis';
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { getSafeJson } from '@openpanel/json';
import type { IClickhouseProfile } from '../services/profile.service';
// Mock chQuery to avoid hitting real ClickHouse
@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
}
beforeEach(async () => {
- await redis.flushdb();
+ const keys = [
+ ...await redis.keys('profile*'),
+ ...await redis.keys('lock:profile'),
+ ];
+ if (keys.length > 0) await redis.del(...keys);
vi.mocked(chQuery).mockResolvedValue([]);
});
@@ -63,89 +66,139 @@ describe('ProfileBuffer', () => {
expect(sizeAfter).toBe(sizeBefore + 1);
});
- it('merges subsequent updates via cache (sequential calls)', async () => {
+ it('concurrent adds: both raw profiles are queued', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
groups: [],
});
+ const groupProfile = makeProfile({
+ first_name: '',
+ email: '',
+ groups: ['group-abc'],
+ });
+
+ const sizeBefore = await profileBuffer.getBufferSize();
+ await Promise.all([
+ profileBuffer.add(identifyProfile),
+ profileBuffer.add(groupProfile),
+ ]);
+ const sizeAfter = await profileBuffer.getBufferSize();
+ // Both raw profiles are queued; merge happens at flush time
+ expect(sizeAfter).toBe(sizeBefore + 2);
+ });
+
+ it('merges sequential updates for the same profile at flush time', async () => {
+ const identifyProfile = makeProfile({
+ first_name: 'John',
+ email: '[email protected]',
+ groups: [],
+ });
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
- // Sequential: identify first, then group
await profileBuffer.add(identifyProfile);
await profileBuffer.add(groupProfile);
+ await profileBuffer.processBuffer();
- // Second add should read the cached identify profile and merge groups in
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('[email protected]');
expect(cached?.groups).toContain('group-abc');
});
- it('race condition: concurrent identify + group calls preserve all data', async () => {
+ it('merges concurrent updates for the same profile at flush time', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
groups: [],
});
-
const groupProfile = makeProfile({
first_name: '',
email: '',
groups: ['group-abc'],
});
- // Both calls run concurrently — the per-profile lock serializes them so the
- // second one reads the first's result from cache and merges correctly.
await Promise.all([
profileBuffer.add(identifyProfile),
profileBuffer.add(groupProfile),
]);
+ await profileBuffer.processBuffer();
const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
-
expect(cached?.first_name).toBe('John');
expect(cached?.email).toBe('[email protected]');
expect(cached?.groups).toContain('group-abc');
});
- it('race condition: concurrent writes produce one merged buffer entry', async () => {
- const identifyProfile = makeProfile({
- first_name: 'John',
- email: '[email protected]',
- groups: [],
+ it('uses existing ClickHouse data for cache misses when merging', async () => {
+ const existingInClickhouse = makeProfile({
+ first_name: 'Jane',
+ email: '[email protected]',
+ groups: ['existing-group'],
});
+ vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
- const groupProfile = makeProfile({
+ const incomingProfile = makeProfile({
first_name: '',
email: '',
- groups: ['group-abc'],
+ groups: ['new-group'],
});
- const sizeBefore = await profileBuffer.getBufferSize();
+ await profileBuffer.add(incomingProfile);
+ await profileBuffer.processBuffer();
- await Promise.all([
- profileBuffer.add(identifyProfile),
- profileBuffer.add(groupProfile),
- ]);
+ const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');
+ expect(cached?.first_name).toBe('Jane');
+ expect(cached?.email).toBe('[email protected]');
+ expect(cached?.groups).toContain('existing-group');
+ expect(cached?.groups).toContain('new-group');
+ });
- const sizeAfter = await profileBuffer.getBufferSize();
+ it('buffer is empty after flush', async () => {
+ await profileBuffer.add(makeProfile({ first_name: 'John' }));
+ expect(await profileBuffer.getBufferSize()).toBe(1);
- // The second add merges into the first — only 2 buffer entries total
- // (one from identify, one merged update with group)
- expect(sizeAfter).toBe(sizeBefore + 2);
+ await profileBuffer.processBuffer();
+
+ expect(await profileBuffer.getBufferSize()).toBe(0);
+ });
+
+ it('retains profiles in queue when ClickHouse insert fails', async () => {
+ await profileBuffer.add(makeProfile({ first_name: 'John' }));
+
+ const { ch } = await import('../clickhouse/client');
+ const insertSpy = vi
+ .spyOn(ch, 'insert')
+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+ await profileBuffer.processBuffer();
+
+ // Profiles must still be in the queue — not lost
+ expect(await profileBuffer.getBufferSize()).toBe(1);
+
+ insertSpy.mockRestore();
+ });
+
+ it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
+ vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+ const { ch } = await import('../clickhouse/client');
+ const insertSpy = vi
+ .spyOn(ch, 'insert')
+ .mockResolvedValueOnce(undefined as any);
+
+ await profileBuffer.add(makeProfile({ first_name: 'John' }));
+ await profileBuffer.processBuffer();
- // The last entry in the buffer should have both name and group
- const rawEntries = await redis.lrange('profile-buffer', 0, -1);
- const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));
- const lastEntry = entries[entries.length - 1];
+ // Insert must still have been called — no data loss even when fetch fails
+ expect(insertSpy).toHaveBeenCalled();
+ expect(await profileBuffer.getBufferSize()).toBe(0);
- expect(lastEntry?.first_name).toBe('John');
- expect(lastEntry?.groups).toContain('group-abc');
+ insertSpy.mockRestore();
});
});
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index 6ff7c00be..e968951c7 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -1,9 +1,6 @@
import { deepMergeObjects } from '@openpanel/common';
-import { generateSecureId } from '@openpanel/common/server';
import { getSafeJson } from '@openpanel/json';
-import type { ILogger } from '@openpanel/logger';
import { getRedisCache, type Redis } from '@openpanel/redis';
-import shallowEqual from 'fast-deep-equal';
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';
import { BaseBuffer } from './base-buffer';
export class ProfileBuffer extends BaseBuffer {
- private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
+ private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
: 200;
- private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
+ private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)
: 1000;
- private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
+ private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS
? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10)
: 60 * 60;
+ /** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */
+ private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE
+ ? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)
+ : 50;
private readonly redisKey = 'profile-buffer';
private readonly redisProfilePrefix = 'profile-cache:';
- private redis: Redis;
- private releaseLockSha: string | null = null;
-
- private readonly releaseLockScript = `
- if redis.call("get", KEYS[1]) == ARGV[1] then
- return redis.call("del", KEYS[1])
- else
- return 0
- end
- `;
+ private readonly redis: Redis;
constructor() {
super({
@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {
},
});
this.redis = getRedisCache();
- this.redis.script('LOAD', this.releaseLockScript).then((sha) => {
- this.releaseLockSha = sha as string;
- });
}
private getProfileCacheKey({
@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {
return `${this.redisProfilePrefix}${projectId}:${profileId}`;
}
- private async withProfileLock<T>(
+ public async fetchFromCache(
profileId: string,
- projectId: string,
- fn: () => Promise<T>
- ): Promise<T> {
- const lockKey = `profile-lock:${projectId}:${profileId}`;
- const lockId = generateSecureId('lock');
- const maxRetries = 20;
- const retryDelayMs = 50;
-
- for (let i = 0; i < maxRetries; i++) {
- const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');
- if (acquired === 'OK') {
- try {
- return await fn();
- } finally {
- if (this.releaseLockSha) {
- await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);
- } else {
- await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);
- }
- }
- }
- await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+ projectId: string
+ ): Promise<IClickhouseProfile | null> {
+ const cacheKey = this.getProfileCacheKey({ profileId, projectId });
+ const cached = await this.redis.get(cacheKey);
+ if (!cached) {
+ return null;
}
-
- this.logger.error(
- 'Failed to acquire profile lock, proceeding without lock',
- {
- profileId,
- projectId,
- }
- );
- return fn();
- }
-
- async alreadyExists(profile: IClickhouseProfile) {
- const cacheKey = this.getProfileCacheKey({
- profileId: profile.id,
- projectId: profile.project_id,
- });
- return (await this.redis.exists(cacheKey)) === 1;
+ return getSafeJson<IClickhouseProfile>(cached);
}
async add(profile: IClickhouseProfile, isFromEvent = false) {
- const logger = this.logger.child({
- projectId: profile.project_id,
- profileId: profile.id,
- });
-
try {
- logger.debug('Adding profile');
-
- if (isFromEvent && (await this.alreadyExists(profile))) {
- logger.debug('Profile already created, skipping');
- return;
- }
-
- await this.withProfileLock(profile.id, profile.project_id, async () => {
- const existingProfile = await this.fetchProfile(profile, logger);
-
- // Delete any properties that are not server related if we have a non-server profile
- if (
- existingProfile?.properties.device !== 'server' &&
- profile.properties.device === 'server'
- ) {
- profile.properties = omit(
- [
- 'city',
- 'country',
- 'region',
- 'longitude',
- 'latitude',
- 'os',
- 'osVersion',
- 'browser',
- 'device',
- 'isServer',
- 'os_version',
- 'browser_version',
- ],
- profile.properties
- );
- }
-
- const mergedProfile: IClickhouseProfile = existingProfile
- ? {
- ...deepMergeObjects(
- existingProfile,
- omit(['created_at', 'groups'], profile)
- ),
- groups: uniq([
- ...(existingProfile.groups ?? []),
- ...(profile.groups ?? []),
- ]),
- }
- : profile;
-
- if (
- profile &&
- existingProfile &&
- shallowEqual(
- omit(['created_at'], existingProfile),
- omit(['created_at'], mergedProfile)
- )
- ) {
- this.logger.debug('Profile not changed, skipping');
- return;
- }
-
- this.logger.debug('Merged profile will be inserted', {
- mergedProfile,
- existingProfile,
- profile,
- });
-
+ if (isFromEvent) {
const cacheKey = this.getProfileCacheKey({
profileId: profile.id,
projectId: profile.project_id,
});
-
- const result = await this.redis
- .multi()
- .set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)
- .rpush(this.redisKey, JSON.stringify(mergedProfile))
- .incr(this.bufferCounterKey)
- .llen(this.redisKey)
- .exec();
-
- if (!result) {
- this.logger.error('Failed to add profile to Redis', {
- profile,
- cacheKey,
- });
+ const exists = await this.redis.exists(cacheKey);
+ if (exists === 1) {
return;
}
- const bufferLength = (result?.[3]?.[1] as number) ?? 0;
+ }
- this.logger.debug('Current buffer length', {
- bufferLength,
- batchSize: this.batchSize,
- });
- if (bufferLength >= this.batchSize) {
- await this.tryFlush();
- }
- });
+ const result = await this.redis
+ .multi()
+ .rpush(this.redisKey, JSON.stringify(profile))
+ .incr(this.bufferCounterKey)
+ .llen(this.redisKey)
+ .exec();
+
+ if (!result) {
+ this.logger.error('Failed to add profile to Redis', { profile });
+ return;
+ }
+
+ const bufferLength = (result?.[2]?.[1] as number) ?? 0;
+ if (bufferLength >= this.batchSize) {
+ await this.tryFlush();
+ }
} catch (error) {
this.logger.error('Failed to add profile', { error, profile });
}
}
- private async fetchProfile(
- profile: IClickhouseProfile,
- logger: ILogger
- ): Promise<IClickhouseProfile | null> {
- const existingProfile = await this.fetchFromCache(
- profile.id,
- profile.project_id
- );
- if (existingProfile) {
- logger.debug('Profile found in Redis');
- return existingProfile;
+ private mergeProfiles(
+ existing: IClickhouseProfile | null,
+ incoming: IClickhouseProfile
+ ): IClickhouseProfile {
+ if (!existing) {
+ return incoming;
}
- return this.fetchFromClickhouse(profile, logger);
- }
-
- public async fetchFromCache(
- profileId: string,
- projectId: string
- ): Promise<IClickhouseProfile | null> {
- const cacheKey = this.getProfileCacheKey({
- profileId,
- projectId,
- });
- const existingProfile = await this.redis.get(cacheKey);
- if (!existingProfile) {
- return null;
+ let profile = incoming;
+ if (
+ existing.properties.device !== 'server' &&
+ incoming.properties.device === 'server'
+ ) {
+ profile = {
+ ...incoming,
+ properties: omit(
+ [
+ 'city',
+ 'country',
+ 'region',
+ 'longitude',
+ 'latitude',
+ 'os',
+ 'osVersion',
+ 'browser',
+ 'device',
+ 'isServer',
+ 'os_version',
+ 'browser_version',
+ ],
+ incoming.properties
+ ),
+ };
}
- return getSafeJson<IClickhouseProfile>(existingProfile);
+
+ return {
+ ...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),
+ groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),
+ };
}
- private async fetchFromClickhouse(
- profile: IClickhouseProfile,
- logger: ILogger
- ): Promise<IClickhouseProfile | null> {
- logger.debug('Fetching profile from Clickhouse');
- const result = await chQuery<IClickhouseProfile>(
- `SELECT
- id,
- project_id,
- last_value(nullIf(first_name, '')) as first_name,
- last_value(nullIf(last_name, '')) as last_name,
- last_value(nullIf(email, '')) as email,
- last_value(nullIf(avatar, '')) as avatar,
- last_value(is_external) as is_external,
- last_value(properties) as properties,
- last_value(created_at) as created_at
- FROM ${TABLE_NAMES.profiles}
- WHERE
- id = ${sqlstring.escape(String(profile.id))} AND
- project_id = ${sqlstring.escape(profile.project_id)}
- ${
- profile.is_external === false
- ? ' AND profiles.created_at > now() - INTERVAL 2 DAY'
- : ''
+ private async batchFetchFromClickhouse(
+ profiles: IClickhouseProfile[]
+ ): Promise<Map<string, IClickhouseProfile>> {
+ const result = new Map<string, IClickhouseProfile>();
+
+ // Non-external (anonymous/device) profiles get a 2-day recency filter to
+ // avoid pulling stale anonymous sessions from far back.
+ const external = profiles.filter((p) => p.is_external !== false);
+ const nonExternal = profiles.filter((p) => p.is_external === false);
+
+ const fetchGroup = async (
+ group: IClickhouseProfile[],
+ withDateFilter: boolean
+ ) => {
+ for (const chunk of this.chunks(group, this.fetchChunkSize)) {
+ const tuples = chunk
+ .map(
+ (p) =>
+ `(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`
+ )
+ .join(', ');
+ try {
+ const rows = await chQuery<IClickhouseProfile>(
+ `SELECT
+ id,
+ project_id,
+ argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,
+ argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,
+ argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,
+ argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,
+ argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,
+ argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,
+ max(created_at) as created_at
+ FROM ${TABLE_NAMES.profiles}
+ WHERE (id, project_id) IN (${tuples})
+ ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}
+ GROUP BY id, project_id`
+ );
+ for (const row of rows) {
+ result.set(`${row.project_id}:${row.id}`, row);
+ }
+ } catch (error) {
+ this.logger.warn(
+ 'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',
+ { error, chunkSize: chunk.length }
+ );
}
- GROUP BY id, project_id
- ORDER BY created_at DESC
- LIMIT 1`
- );
- logger.debug('Clickhouse fetch result', {
- found: !!result[0],
- });
- return result[0] || null;
+ }
+ };
+
+ await Promise.all([
+ fetchGroup(external, false),
+ fetchGroup(nonExternal, true),
+ ]);
+
+ return result;
}
async processBuffer() {
try {
this.logger.debug('Starting profile buffer processing');
- const profiles = await this.redis.lrange(
+ const rawProfiles = await this.redis.lrange(
this.redisKey,
0,
this.batchSize - 1
);
- if (profiles.length === 0) {
+ if (rawProfiles.length === 0) {
this.logger.debug('No profiles to process');
return;
}
- this.logger.debug(`Processing ${profiles.length} profiles in buffer`);
- const parsedProfiles = profiles.map((p) =>
- getSafeJson<IClickhouseProfile>(p)
+ const parsedProfiles = rawProfiles
+ .map((p) => getSafeJson<IClickhouseProfile>(p))
+ .filter(Boolean) as IClickhouseProfile[];
+
+ // Merge within batch: collapse multiple updates for the same profile
+ const mergedInBatch = new Map<string, IClickhouseProfile>();
+ for (const profile of parsedProfiles) {
+ const key = `${profile.project_id}:${profile.id}`;
+ mergedInBatch.set(
+ key,
+ this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
+ );
+ }
+
+ const uniqueProfiles = Array.from(mergedInBatch.values());
+
+ // Check Redis cache for all unique profiles in a single MGET
+ const cacheKeys = uniqueProfiles.map((p) =>
+ this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id })
);
+ const cacheResults = await this.redis.mget(...cacheKeys);
+
+ const existingByKey = new Map<string, IClickhouseProfile>();
+ const cacheMisses: IClickhouseProfile[] = [];
+ for (let i = 0; i < uniqueProfiles.length; i++) {
+ const uniqueProfile = uniqueProfiles[i];
+ if (uniqueProfile) {
+ const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;
+ const cached = cacheResults[i]
+ ? getSafeJson<IClickhouseProfile>(cacheResults[i]!)
+ : null;
+ if (cached) {
+ existingByKey.set(key, cached);
+ } else {
+ cacheMisses.push(uniqueProfile);
+ }
+ }
+ }
+
+ // Fetch cache misses from ClickHouse in bounded chunks
+ if (cacheMisses.length > 0) {
+ const clickhouseResults =
+ await this.batchFetchFromClickhouse(cacheMisses);
+ for (const [key, profile] of clickhouseResults) {
+ existingByKey.set(key, profile);
+ }
+ }
- for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {
+ // Final merge: in-batch profile + existing (from cache or ClickHouse)
+ const toInsert: IClickhouseProfile[] = [];
+ const multi = this.redis.multi();
+
+ for (const profile of uniqueProfiles) {
+ const key = `${profile.project_id}:${profile.id}`;
+ const merged = this.mergeProfiles(
+ existingByKey.get(key) ?? null,
+ profile
+ );
+ toInsert.push(merged);
+ multi.set(
+ this.getProfileCacheKey({
+ projectId: profile.project_id,
+ profileId: profile.id,
+ }),
+ JSON.stringify(merged),
+ 'EX',
+ this.ttlInSeconds
+ );
+ }
+
+ for (const chunk of this.chunks(toInsert, this.chunkSize)) {
await ch.insert({
table: TABLE_NAMES.profiles,
values: chunk,
@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {
});
}
- // Only remove profiles after successful insert and update counter
- await this.redis
- .multi()
- .ltrim(this.redisKey, profiles.length, -1)
- .decrby(this.bufferCounterKey, profiles.length)
- .exec();
+ multi
+ .ltrim(this.redisKey, rawProfiles.length, -1)
+ .decrby(this.bufferCounterKey, rawProfiles.length);
+ await multi.exec();
this.logger.debug('Successfully completed profile processing', {
- totalProfiles: profiles.length,
+ totalProfiles: rawProfiles.length,
+ uniqueProfiles: uniqueProfiles.length,
});
} catch (error) {
this.logger.error('Failed to process buffer', { error });
}
}
- async getBufferSize() {
+ getBufferSize() {
return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));
}
}
diff --git a/packages/db/src/buffers/session-buffer.test.ts b/packages/db/src/buffers/session-buffer.test.ts
new file mode 100644
index 000000000..f140ff002
--- /dev/null
+++ b/packages/db/src/buffers/session-buffer.test.ts
@@ -0,0 +1,122 @@
+import { getRedisCache } from '@openpanel/redis';
+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { ch } from '../clickhouse/client';
+
+vi.mock('../clickhouse/client', () => ({
+ ch: {
+ insert: vi.fn().mockResolvedValue(undefined),
+ },
+ TABLE_NAMES: {
+ sessions: 'sessions',
+ },
+}));
+
+import { SessionBuffer } from './session-buffer';
+import type { IClickhouseEvent } from '../services/event.service';
+
+const redis = getRedisCache();
+
+function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {
+ return {
+ id: 'event-1',
+ project_id: 'project-1',
+ profile_id: 'profile-1',
+ device_id: 'device-1',
+ session_id: 'session-1',
+ name: 'screen_view',
+ path: '/home',
+ origin: '',
+ referrer: '',
+ referrer_name: '',
+ referrer_type: '',
+ duration: 0,
+ properties: {},
+ created_at: new Date().toISOString(),
+ groups: [],
+ ...overrides,
+ } as IClickhouseEvent;
+}
+
+beforeEach(async () => {
+ const keys = [
+ ...await redis.keys('session*'),
+ ...await redis.keys('lock:session'),
+ ];
+ if (keys.length > 0) await redis.del(...keys);
+ vi.mocked(ch.insert).mockResolvedValue(undefined as any);
+});
+
+afterAll(async () => {
+ try {
+ await redis.quit();
+ } catch {}
+});
+
+describe('SessionBuffer', () => {
+ let sessionBuffer: SessionBuffer;
+
+ beforeEach(() => {
+ sessionBuffer = new SessionBuffer();
+ });
+
+ it('adds a new session to the buffer', async () => {
+ const sizeBefore = await sessionBuffer.getBufferSize();
+ await sessionBuffer.add(makeEvent({}));
+ const sizeAfter = await sessionBuffer.getBufferSize();
+
+ expect(sizeAfter).toBe(sizeBefore + 1);
+ });
+
+ it('skips session_start and session_end events', async () => {
+ const sizeBefore = await sessionBuffer.getBufferSize();
+ await sessionBuffer.add(makeEvent({ name: 'session_start' }));
+ await sessionBuffer.add(makeEvent({ name: 'session_end' }));
+ const sizeAfter = await sessionBuffer.getBufferSize();
+
+ expect(sizeAfter).toBe(sizeBefore);
+ });
+
+ it('updates existing session on subsequent events', async () => {
+ const t0 = Date.now();
+ await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));
+
+ // Second event updates the same session — emits old (sign=-1) + new (sign=1)
+ const sizeBefore = await sessionBuffer.getBufferSize();
+ await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));
+ const sizeAfter = await sessionBuffer.getBufferSize();
+
+ expect(sizeAfter).toBe(sizeBefore + 2);
+ });
+
+ it('processes buffer and inserts sessions into ClickHouse', async () => {
+ await sessionBuffer.add(makeEvent({}));
+
+ const insertSpy = vi
+ .spyOn(ch, 'insert')
+ .mockResolvedValueOnce(undefined as any);
+
+ await sessionBuffer.processBuffer();
+
+ expect(insertSpy).toHaveBeenCalledWith(
+ expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
+ );
+ expect(await sessionBuffer.getBufferSize()).toBe(0);
+
+ insertSpy.mockRestore();
+ });
+
+ it('retains sessions in queue when ClickHouse insert fails', async () => {
+ await sessionBuffer.add(makeEvent({}));
+
+ const insertSpy = vi
+ .spyOn(ch, 'insert')
+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+
+ await sessionBuffer.processBuffer();
+
+ // Sessions must still be in the queue — not lost
+ expect(await sessionBuffer.getBufferSize()).toBe(1);
+
+ insertSpy.mockRestore();
+ });
+});
diff --git a/packages/redis/cachable.test.ts b/packages/redis/cachable.test.ts
index d628f4391..2d212a2ea 100644
--- a/packages/redis/cachable.test.ts
+++ b/packages/redis/cachable.test.ts
@@ -8,7 +8,10 @@ describe('cachable', () => {
beforeEach(async () => {
redis = getRedisCache();
// Clear any existing cache data for clean tests
- const keys = await redis.keys('cachable:*');
+ const keys = [
+ ...await redis.keys('cachable:*'),
+ ...await redis.keys('test-key*'),
+ ];
if (keys.length > 0) {
await redis.del(...keys);
}
@@ -16,7 +19,10 @@ describe('cachable', () => {
afterEach(async () => {
// Clean up after each test
- const keys = await redis.keys('cachable:*');
+ const keys = [
+ ...await redis.keys('cachable:*'),
+ ...await redis.keys('test-key*'),
+ ];
if (keys.length > 0) {
await redis.del(...keys);
}