OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix:buffers
Carl-Gerhard Lindesvärda1ce71ffb660
diff --git a/apps/api/src/controllers/live.controller.ts b/apps/api/src/controllers/live.controller.tsindex 488e67138..d73f267ec 100644--- a/apps/api/src/controllers/live.controller.ts+++ b/apps/api/src/controllers/live.controller.ts@@ -1,10 +1,7 @@import type { WebSocket } from '@fastify/websocket';import { eventBuffer } from '@openpanel/db';import { setSuperJson } from '@openpanel/json';-import {- psubscribeToPublishedEvent,- subscribeToPublishedEvent,-} from '@openpanel/redis';+import { subscribeToPublishedEvent } from '@openpanel/redis';import { getProjectAccess } from '@openpanel/trpc';import { getOrganizationAccess } from '@openpanel/trpc/src/access';import type { FastifyRequest } from 'fastify';@@ -39,19 +36,8 @@ export function wsVisitors(});- const punsubscribe = psubscribeToPublishedEvent(- '__keyevent@0__:expired',- (key) => {- const [, , projectId] = key.split(':');- if (projectId === params.projectId) {- sendCount();- }- }- );-socket.on('close', () => {unsubscribe();- punsubscribe();});}diff --git a/apps/start/src/components/overview/live-counter.tsx b/apps/start/src/components/overview/live-counter.tsxindex 2333f5c09..c3d5e7336 100644--- a/apps/start/src/components/overview/live-counter.tsx+++ b/apps/start/src/components/overview/live-counter.tsx@@ -1,61 +1,25 @@-import { TooltipComplete } from '@/components/tooltip-complete';-import { useDebounceState } from '@/hooks/use-debounce-state';-import useWS from '@/hooks/use-ws';-import { useTRPC } from '@/integrations/trpc/react';-import { cn } from '@/utils/cn';-import { useQuery, useQueryClient } from '@tanstack/react-query';-import { useEffect, useRef } from 'react';+import { useQueryClient } from '@tanstack/react-query';+import { useCallback } from 'react';import { toast } from 'sonner';import { AnimatedNumber } from '../animated-number';+import { TooltipComplete } from '@/components/tooltip-complete';+import { useLiveCounter } from '@/hooks/use-live-counter';+import { cn } from '@/utils/cn';export interface LiveCounterProps {projectId: string;shareId?: string;}-const FIFTEEN_SECONDS = 1000 * 30;-export function LiveCounter({ projectId, shareId }: LiveCounterProps) {- const trpc = useTRPC();const client = useQueryClient();- const counter = useDebounceState(0, 1000);- const lastRefresh = useRef(Date.now());- const query = useQuery(- trpc.overview.liveVisitors.queryOptions({- projectId,- shareId,- }),- );-- useEffect(() => {- if (query.data) {- counter.set(query.data);- }- }, [query.data]);-- useWS<number>(- `/live/visitors/${projectId}`,- (value) => {- if (!Number.isNaN(value)) {- counter.set(value);- if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {- lastRefresh.current = Date.now();- if (!document.hidden) {- toast('Refreshed data');- client.refetchQueries({- type: 'active',- });- }- }- }- },- {- debounce: {- delay: 1000,- maxWait: 5000,- },- },- );+ const onRefresh = useCallback(() => {+ toast('Refreshed data');+ client.refetchQueries({+ type: 'active',+ });+ }, [client]);+ const counter = useLiveCounter({ projectId, shareId, onRefresh });return (<TooltipComplete@@ -66,13 +30,13 @@ export function LiveCounter({ projectId, shareId }: LiveCounterProps) {<divclassName={cn('h-3 w-3 animate-ping rounded-full bg-emerald-500 opacity-100 transition-all',- counter.debounced === 0 && 'bg-destructive opacity-0',+ counter.debounced === 0 && 'bg-destructive opacity-0')}/><divclassName={cn(- 'absolute left-0 top-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',- counter.debounced === 0 && 'bg-destructive',+ 'absolute top-0 left-0 h-3 w-3 rounded-full bg-emerald-500 transition-all',+ counter.debounced === 0 && 'bg-destructive')}/></div>diff --git a/apps/start/src/hooks/use-live-counter.ts b/apps/start/src/hooks/use-live-counter.tsnew file mode 100644index 000000000..d8e0d7eda--- /dev/null+++ b/apps/start/src/hooks/use-live-counter.ts@@ -0,0 +1,81 @@+import { useQuery, useQueryClient } from '@tanstack/react-query';+import { useEffect, useRef } from 'react';+import { useDebounceState } from './use-debounce-state';+import useWS from './use-ws';+import { useTRPC } from '@/integrations/trpc/react';++const FIFTEEN_SECONDS = 1000 * 15;+/** Refetch from API when WS-only updates may be stale (e.g. visitors left). */+const FALLBACK_STALE_MS = 1000 * 60;++export function useLiveCounter({+ projectId,+ shareId,+ onRefresh,+}: {+ projectId: string;+ shareId?: string;+ onRefresh?: () => void;+}) {+ const trpc = useTRPC();+ const queryClient = useQueryClient();+ const counter = useDebounceState(0, 1000);+ const lastRefresh = useRef(Date.now());+ const query = useQuery(+ trpc.overview.liveVisitors.queryOptions({+ projectId,+ shareId: shareId ?? undefined,+ })+ );++ useEffect(() => {+ if (query.data) {+ counter.set(query.data);+ }+ }, [query.data]);++ useWS<number>(+ `/live/visitors/${projectId}`,+ (value) => {+ if (!Number.isNaN(value)) {+ counter.set(value);+ if (Date.now() - lastRefresh.current > FIFTEEN_SECONDS) {+ lastRefresh.current = Date.now();+ if (!document.hidden) {+ onRefresh?.();+ }+ }+ }+ },+ {+ debounce: {+ delay: 1000,+ maxWait: 5000,+ },+ }+ );++ useEffect(() => {+ const id = setInterval(async () => {+ if (Date.now() - lastRefresh.current < FALLBACK_STALE_MS) {+ return;+ }+ const data = await queryClient.fetchQuery(+ trpc.overview.liveVisitors.queryOptions(+ {+ projectId,+ shareId: shareId ?? undefined,+ },+ // Default query staleTime is 5m; bypass cache so this reconciliation always hits the API.+ { staleTime: 0 }+ )+ );+ counter.set(data);+ lastRefresh.current = Date.now();+ }, FALLBACK_STALE_MS);++ return () => clearInterval(id);+ }, [projectId, shareId, trpc, queryClient, counter.set]);++ return counter;+}diff --git a/apps/start/src/routes/widget/counter.tsx b/apps/start/src/routes/widget/counter.tsxindex a55e7c81a..d745aa62e 100644--- a/apps/start/src/routes/widget/counter.tsx+++ b/apps/start/src/routes/widget/counter.tsx@@ -1,12 +1,11 @@+import { useQuery, useQueryClient } from '@tanstack/react-query';+import { createFileRoute } from '@tanstack/react-router';+import { z } from 'zod';import { AnimatedNumber } from '@/components/animated-number';import { Ping } from '@/components/ping';-import { useNumber } from '@/hooks/use-numer-formatter';import useWS from '@/hooks/use-ws';import { useTRPC } from '@/integrations/trpc/react';import type { RouterOutputs } from '@/trpc/client';-import { useQuery, useQueryClient } from '@tanstack/react-query';-import { createFileRoute } from '@tanstack/react-router';-import { z } from 'zod';const widgetSearchSchema = z.object({shareId: z.string(),@@ -20,33 +19,33 @@ export const Route = createFileRoute('/widget/counter')({});function RouteComponent() {- const { shareId, limit, color } = Route.useSearch();+ const { shareId } = Route.useSearch();const trpc = useTRPC();// Fetch widget dataconst { data, isLoading } = useQuery(- trpc.widget.counter.queryOptions({ shareId }),+ trpc.widget.counter.queryOptions({ shareId }));if (isLoading) {return (- <div className="flex items-center gap-2 px-2 h-8">+ <div className="flex h-8 items-center gap-2 px-2"><Ping />- <AnimatedNumber value={0} suffix=" unique visitors" />+ <AnimatedNumber suffix=" unique visitors" value={0} /></div>);}if (!data) {return (- <div className="flex items-center gap-2 px-2 h-8">+ <div className="flex h-8 items-center gap-2 px-2"><Ping className="bg-orange-500" />- <AnimatedNumber value={0} suffix=" unique visitors" />+ <AnimatedNumber suffix=" unique visitors" value={0} /></div>);}- return <CounterWidget shareId={shareId} data={data} />;+ return <CounterWidget data={data} shareId={shareId} />;}interface RealtimeWidgetProps {@@ -57,30 +56,29 @@ interface RealtimeWidgetProps {function CounterWidget({ shareId, data }: RealtimeWidgetProps) {const trpc = useTRPC();const queryClient = useQueryClient();- const number = useNumber();// WebSocket subscription for real-time updatesuseWS<number>(`/live/visitors/${data.projectId}`,- (res) => {+ () => {if (!document.hidden) {queryClient.refetchQueries(- trpc.widget.counter.queryFilter({ shareId }),+ trpc.widget.counter.queryFilter({ shareId }));}},{debounce: {delay: 1000,- maxWait: 60000,+ maxWait: 60_000,},- },+ });return (- <div className="flex items-center gap-2 px-2 h-8">+ <div className="flex h-8 items-center gap-2 px-2"><Ping />- <AnimatedNumber value={data.counter} suffix=" unique visitors" />+ <AnimatedNumber suffix=" unique visitors" value={data.counter} /></div>);}diff --git a/apps/start/src/routes/widget/realtime.tsx b/apps/start/src/routes/widget/realtime.tsxindex a60a17a8e..8e1a4cdb3 100644--- a/apps/start/src/routes/widget/realtime.tsx+++ b/apps/start/src/routes/widget/realtime.tsx@@ -1,3 +1,15 @@+import { useQuery, useQueryClient } from '@tanstack/react-query';+import { createFileRoute } from '@tanstack/react-router';+import type React from 'react';+import {+ Bar,+ BarChart,+ ResponsiveContainer,+ Tooltip,+ XAxis,+ YAxis,+} from 'recharts';+import { z } from 'zod';import { AnimatedNumber } from '@/components/animated-number';import {ChartTooltipContainer,@@ -14,18 +26,6 @@ import { countries } from '@/translations/countries';import type { RouterOutputs } from '@/trpc/client';import { cn } from '@/utils/cn';import { getChartColor } from '@/utils/theme';-import { useQuery, useQueryClient } from '@tanstack/react-query';-import { createFileRoute } from '@tanstack/react-router';-import type React from 'react';-import {- Bar,- BarChart,- ResponsiveContainer,- Tooltip,- XAxis,- YAxis,-} from 'recharts';-import { z } from 'zod';const widgetSearchSchema = z.object({shareId: z.string(),@@ -44,7 +44,7 @@ function RouteComponent() {// Fetch widget dataconst { data: widgetData, isLoading } = useQuery(- trpc.widget.realtimeData.queryOptions({ shareId }),+ trpc.widget.realtimeData.queryOptions({ shareId }));if (isLoading) {@@ -53,10 +53,10 @@ function RouteComponent() {if (!widgetData) {return (- <div className="flex h-screen w-full center-center bg-background text-foreground col p-4">- <LogoSquare className="size-10 mb-4" />- <h1 className="text-xl font-semibold">Widget not found</h1>- <p className="mt-2 text-sm text-muted-foreground">+ <div className="center-center col flex h-screen w-full bg-background p-4 text-foreground">+ <LogoSquare className="mb-4 size-10" />+ <h1 className="font-semibold text-xl">Widget not found</h1>+ <p className="mt-2 text-muted-foreground text-sm">This widget is not available or has been removed.</p></div>@@ -65,10 +65,10 @@ function RouteComponent() {return (<RealtimeWidget- shareId={shareId}- limit={limit}- data={widgetData}color={color}+ data={widgetData}+ limit={limit}+ shareId={shareId}/>);}@@ -83,7 +83,6 @@ interface RealtimeWidgetProps {function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {const trpc = useTRPC();const queryClient = useQueryClient();- const number = useNumber();// WebSocket subscription for real-time updatesuseWS<number>(@@ -91,16 +90,16 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {() => {if (!document.hidden) {queryClient.refetchQueries(- trpc.widget.realtimeData.queryFilter({ shareId }),+ trpc.widget.realtimeData.queryFilter({ shareId }));}},{debounce: {delay: 1000,- maxWait: 60000,+ maxWait: 60_000,},- },+ });const maxDomain =@@ -111,8 +110,12 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {const referrers = data.referrers.length > 0 ? 1 : 0;const paths = data.paths.length > 0 ? 1 : 0;const value = countries + referrers + paths;- if (value === 3) return 'md:grid-cols-3';- if (value === 2) return 'md:grid-cols-2';+ if (value === 3) {+ return 'md:grid-cols-3';+ }+ if (value === 2) {+ return 'md:grid-cols-2';+ }return 'md:grid-cols-1';})();@@ -120,10 +123,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {<div className="flex h-screen w-full flex-col bg-background text-foreground">{/* Header with live counter */}<div className="p-6 pb-3">- <div className="flex items-center justify-between w-full h-4">- <div className="flex items-center gap-3 w-full">+ <div className="flex h-4 w-full items-center justify-between">+ <div className="flex w-full items-center gap-3"><Ping />- <div className="text-sm font-medium text-muted-foreground flex-1">+ <div className="flex-1 font-medium text-muted-foreground text-sm">USERS IN LAST 30 MINUTES</div>{data.project.domain && <SerieIcon name={data.project.domain} />}@@ -131,14 +134,14 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {</div><div className="row">- <div className="font-mono text-6xl font-bold h-18 text-foreground">+ <div className="h-18 font-bold font-mono text-6xl text-foreground"><AnimatedNumber value={data.liveCount} /></div></div>- <div className="flex h-20 w-full flex-col -mt-4">+ <div className="-mt-4 flex h-20 w-full flex-col"><div className="flex-1">- <ResponsiveContainer width="100%" height="100%">+ <ResponsiveContainer height="100%" width="100%"><BarChartdata={data.histogram}margin={{ top: 0, right: 0, left: 0, bottom: 0 }}@@ -148,22 +151,22 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {cursor={{ fill: 'var(--def-100)', radius: 4 }}/><XAxis- dataKey="time"axisLine={false}- tickLine={false}+ dataKey="time"+ interval="preserveStartEnd"tick={{ fill: 'var(--muted-foreground)', fontSize: 10 }}+ tickLine={false}ticks={[data.histogram[0].time,data.histogram[data.histogram.length - 1].time,]}- interval="preserveStartEnd"/>- <YAxis hide domain={[0, maxDomain]} />+ <YAxis domain={[0, maxDomain]} hide /><BardataKey="sessionCount"+ fill={color || 'var(--chart-0)'}isAnimationActive={false}radius={[4, 4, 4, 4]}- fill={color || 'var(--chart-0)'}/></BarChart></ResponsiveContainer>@@ -174,24 +177,24 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {{(data.countries.length > 0 ||data.referrers.length > 0 ||data.paths.length > 0) && (- <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar border-t">+ <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto border-t p-6"><div className={cn('grid grid-cols-1 gap-6', grids)}>{/* Countries */}{data.countries.length > 0 && (<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">COUNTRY</div><div className="col">{(() => {const { visible, rest, restCount } = getRestItems(data.countries,- limit,+ limit);return (<>{visible.map((item) => (- <RowItem key={item.country} count={item.count}>+ <RowItem count={item.count} key={item.country}><div className="flex items-center gap-2"><SerieIcon name={item.country} /><span className="text-sm">@@ -224,19 +227,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {{/* Referrers */}{data.referrers.length > 0 && (<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">REFERRER</div><div className="col">{(() => {const { visible, rest, restCount } = getRestItems(data.referrers,- limit,+ limit);return (<>{visible.map((item) => (- <RowItem key={item.referrer} count={item.count}>+ <RowItem count={item.count} key={item.referrer}><div className="flex items-center gap-2"><SerieIcon name={item.referrer} /><span className="truncate text-sm">@@ -263,19 +266,19 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {{/* Paths */}{data.paths.length > 0 && (<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">PATH</div><div className="col">{(() => {const { visible, rest, restCount } = getRestItems(data.paths,- limit,+ limit);return (<>{visible.map((item) => (- <RowItem key={item.path} count={item.count}>+ <RowItem count={item.count} key={item.path}><span className="truncate text-sm">{item.path}</span>@@ -303,10 +306,10 @@ function RealtimeWidget({ shareId, data, limit, color }: RealtimeWidgetProps) {}// Custom tooltip component that uses portals to escape overflow hidden-const CustomTooltip = ({ active, payload, coordinate }: any) => {+const CustomTooltip = ({ active, payload }: any) => {const number = useNumber();- if (!active || !payload || !payload.length) {+ if (!(active && payload && payload.length)) {return null;}@@ -328,10 +331,13 @@ const CustomTooltip = ({ active, payload, coordinate }: any) => {function RowItem({children,count,-}: { children: React.ReactNode; count: number }) {+}: {+ children: React.ReactNode;+ count: number;+}) {const number = useNumber();return (- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5">{children}<span className="font-semibold">{number.short(count)}</span></div>@@ -340,7 +346,7 @@ function RowItem({function getRestItems<T extends { count: number }>(items: T[],- limit: number,+ limit: number): { visible: T[]; rest: T[]; restCount: number } {const visible = items.slice(0, limit);const rest = items.slice(limit);@@ -375,7 +381,7 @@ function RestRow({: 'paths';return (- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b hover:bg-foreground/5 -mx-3">+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm hover:bg-foreground/5"><span className="truncate">{firstName} and {otherCount} more {typeLabel}...</span>@@ -434,13 +440,13 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {const itemCount = Math.min(limit, 5);return (- <div className="flex h-screen w-full flex-col bg-background text-foreground animate-pulse">+ <div className="flex h-screen w-full animate-pulse flex-col bg-background text-foreground">{/* Header with live counter */}<div className="border-b p-6 pb-3">- <div className="flex items-center justify-between w-full h-4">- <div className="flex items-center gap-3 w-full">+ <div className="flex h-4 w-full items-center justify-between">+ <div className="flex w-full items-center gap-3"><div className="size-2 rounded-full bg-muted" />- <div className="text-sm font-medium text-muted-foreground flex-1">+ <div className="flex-1 font-medium text-muted-foreground text-sm">USERS IN LAST 30 MINUTES</div></div>@@ -448,35 +454,35 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {</div><div className="row">- <div className="font-mono text-6xl font-bold h-18 flex items-center py-4 gap-1 row">- <div className="h-full w-6 bg-muted rounded" />- <div className="h-full w-6 bg-muted rounded" />+ <div className="row flex h-18 items-center gap-1 py-4 font-bold font-mono text-6xl">+ <div className="h-full w-6 rounded bg-muted" />+ <div className="h-full w-6 rounded bg-muted" /></div></div>- <div className="flex h-20 w-full flex-col -mt-4 pb-2.5">- <div className="flex-1 row gap-1 h-full">+ <div className="-mt-4 flex h-20 w-full flex-col pb-2.5">+ <div className="row h-full flex-1 gap-1">{SKELETON_HISTOGRAM.map((item, index) => (<div+ className="mt-auto h-full w-full rounded bg-muted"key={index.toString()}style={{ height: `${item}%` }}- className="h-full w-full bg-muted rounded mt-auto"/>))}</div><div className="row justify-between pt-2">- <div className="h-3 w-8 bg-muted rounded" />- <div className="h-3 w-8 bg-muted rounded" />+ <div className="h-3 w-8 rounded bg-muted" />+ <div className="h-3 w-8 rounded bg-muted" /></div></div></div>- <div className="flex flex-1 flex-col gap-6 overflow-auto p-6 hide-scrollbar">+ <div className="hide-scrollbar flex flex-1 flex-col gap-6 overflow-auto p-6">{/* Countries, Referrers, and Paths skeleton */}<div className="grid grid-cols-1 gap-6 md:grid-cols-3">{/* Countries skeleton */}<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">COUNTRY</div><div className="col">@@ -488,7 +494,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {{/* Referrers skeleton */}<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">REFERRER</div><div className="col">@@ -500,7 +506,7 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {{/* Paths skeleton */}<div className="flex flex-col">- <div className="mb-3 text-xs font-medium text-muted-foreground">+ <div className="mb-3 font-medium text-muted-foreground text-xs">PATH</div><div className="col">@@ -517,12 +523,12 @@ function RealtimeWidgetSkeleton({ limit }: { limit: number }) {function RowItemSkeleton() {return (- <div className="h-10 text-sm flex items-center justify-between px-3 py-2 border-b -mx-3">+ <div className="-mx-3 flex h-10 items-center justify-between border-b px-3 py-2 text-sm"><div className="flex items-center gap-2"><div className="size-5 rounded bg-muted" />- <div className="h-4 w-24 bg-muted rounded" />+ <div className="h-4 w-24 rounded bg-muted" /></div>- <div className="h-4 w-8 bg-muted rounded" />+ <div className="h-4 w-8 rounded bg-muted" /></div>);}diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.tsindex 178f94545..e269ef0b1 100644--- a/packages/db/src/buffers/event-buffer.test.ts+++ b/packages/db/src/buffers/event-buffer.test.ts@@ -1,6 +1,7 @@import { getRedisCache } from '@openpanel/redis';import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';-import { ch } from '../clickhouse/client';+import * as chClient from '../clickhouse/client';+const { ch } = chClient;// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffervi.mock('../services/event.service', () => ({}));@@ -10,7 +11,8 @@ import { EventBuffer } from './event-buffer';const redis = getRedisCache();beforeEach(async () => {- await redis.flushdb();+ const keys = await redis.keys('event*');+ if (keys.length > 0) await redis.del(...keys);});afterAll(async () => {@@ -209,18 +211,16 @@ describe('EventBuffer', () => {});it('tracks active visitors', async () => {- const event = {- project_id: 'p9',- profile_id: 'u9',- name: 'custom',- created_at: new Date().toISOString(),- } as any;-- eventBuffer.add(event);- await eventBuffer.flush();+ const querySpy = vi+ .spyOn(chClient, 'chQuery')+ .mockResolvedValueOnce([{ count: 2 }] as any);const count = await eventBuffer.getActiveVisitorCount('p9');- expect(count).toBeGreaterThanOrEqual(1);+ expect(count).toBe(2);+ expect(querySpy).toHaveBeenCalledOnce();+ expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");++ querySpy.mockRestore();});it('handles multiple sessions independently — all events go to buffer', async () => {@@ -273,4 +273,24 @@ describe('EventBuffer', () => {expect(await eventBuffer.getBufferSize()).toBe(5);});++ it('retains events in queue when ClickHouse insert fails', async () => {+ eventBuffer.add({+ project_id: 'p12',+ name: 'event1',+ created_at: new Date().toISOString(),+ } as any);+ await eventBuffer.flush();++ const insertSpy = vi+ .spyOn(ch, 'insert')+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));++ await eventBuffer.processBuffer();++ // Events must still be in the queue — not lost+ expect(await eventBuffer.getBufferSize()).toBe(1);++ insertSpy.mockRestore();+ });});diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 2e5d43dce..883812adc 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -1,6 +1,6 @@import { getSafeJson } from '@openpanel/json';-import { getRedisCache, publishEvent, type Redis } from '@openpanel/redis';-import { ch } from '../clickhouse/client';+import { getRedisCache, publishEvent } from '@openpanel/redis';+import { ch, chQuery } from '../clickhouse/client';import type { IClickhouseEvent } from '../services/event.service';import { BaseBuffer } from './base-buffer';@@ -25,10 +25,6 @@ export class EventBuffer extends BaseBuffer {/** Tracks consecutive flush failures for observability; reset on success. */private flushRetryCount = 0;- private activeVisitorsExpiration = 60 * 5; // 5 minutes- /** How often (ms) we refresh the heartbeat key + zadd per visitor. */- private heartbeatRefreshMs = 60_000; // 1 minute- private lastHeartbeat = new Map<string, number>();private queueKey = 'event_buffer:queue';protected bufferCounterKey = 'event_buffer:total_count';@@ -87,20 +83,12 @@ export class EventBuffer extends BaseBuffer {for (const event of eventsToFlush) {multi.rpush(this.queueKey, JSON.stringify(event));- if (event.profile_id) {- this.incrementActiveVisitorCount(- multi,- event.project_id,- event.profile_id- );- }}multi.incrby(this.bufferCounterKey, eventsToFlush.length);await multi.exec();this.flushRetryCount = 0;- this.pruneHeartbeatMap();} catch (error) {// Re-queue failed events at the front to preserve order and avoid data lossthis.pendingEvents = eventsToFlush.concat(this.pendingEvents);@@ -202,58 +190,21 @@ export class EventBuffer extends BaseBuffer {}}- public async getBufferSize() {+ public getBufferSize() {return this.getBufferSizeWithCounter(async () => {const redis = getRedisCache();return await redis.llen(this.queueKey);});}- private pruneHeartbeatMap() {- const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;- for (const [key, ts] of this.lastHeartbeat) {- if (ts < cutoff) {- this.lastHeartbeat.delete(key);- }- }- }-- private incrementActiveVisitorCount(- multi: ReturnType<Redis['multi']>,- projectId: string,- profileId: string- ) {- const key = `${projectId}:${profileId}`;- const now = Date.now();- const last = this.lastHeartbeat.get(key) ?? 0;-- if (now - last < this.heartbeatRefreshMs) {- return;- }-- this.lastHeartbeat.set(key, now);- const zsetKey = `live:visitors:${projectId}`;- const heartbeatKey = `live:visitor:${projectId}:${profileId}`;- multi- .zadd(zsetKey, now, profileId)- .set(heartbeatKey, '1', 'EX', this.activeVisitorsExpiration);- }-public async getActiveVisitorCount(projectId: string): Promise<number> {- const redis = getRedisCache();- const zsetKey = `live:visitors:${projectId}`;- const cutoff = Date.now() - this.activeVisitorsExpiration * 1000;-- const multi = redis.multi();- multi- .zremrangebyscore(zsetKey, '-inf', cutoff)- .zcount(zsetKey, cutoff, '+inf');-- const [, count] = (await multi.exec()) as [- [Error | null, any],- [Error | null, number],- ];-- return count[1] || 0;+ const rows = await chQuery<{ count: number }>(+ `SELECT uniq(profile_id) AS count+ FROM events+ WHERE project_id = '${projectId}'+ AND profile_id != ''+ AND created_at >= now() - INTERVAL 5 MINUTE`+ );+ return rows[0]?.count ?? 0;}}diff --git a/packages/db/src/buffers/profile-buffer.test.ts b/packages/db/src/buffers/profile-buffer.test.tsindex bc7d39d22..2dc757f97 100644--- a/packages/db/src/buffers/profile-buffer.test.ts+++ b/packages/db/src/buffers/profile-buffer.test.ts@@ -1,6 +1,5 @@import { getRedisCache } from '@openpanel/redis';import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';-import { getSafeJson } from '@openpanel/json';import type { IClickhouseProfile } from '../services/profile.service';// Mock chQuery to avoid hitting real ClickHouse@@ -36,7 +35,11 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile}beforeEach(async () => {- await redis.flushdb();+ const keys = [+ ...await redis.keys('profile*'),+ ...await redis.keys('lock:profile'),+ ];+ if (keys.length > 0) await redis.del(...keys);vi.mocked(chQuery).mockResolvedValue([]);});@@ -63,89 +66,139 @@ describe('ProfileBuffer', () => {expect(sizeAfter).toBe(sizeBefore + 1);});- it('merges subsequent updates via cache (sequential calls)', async () => {+ it('concurrent adds: both raw profiles are queued', async () => {const identifyProfile = makeProfile({first_name: 'John',email: '[email protected]',groups: [],});+ const groupProfile = makeProfile({+ first_name: '',+ email: '',+ groups: ['group-abc'],+ });++ const sizeBefore = await profileBuffer.getBufferSize();+ await Promise.all([+ profileBuffer.add(identifyProfile),+ profileBuffer.add(groupProfile),+ ]);+ const sizeAfter = await profileBuffer.getBufferSize();+ // Both raw profiles are queued; merge happens at flush time+ expect(sizeAfter).toBe(sizeBefore + 2);+ });++ it('merges sequential updates for the same profile at flush time', async () => {+ const identifyProfile = makeProfile({+ first_name: 'John',+ email: '[email protected]',+ groups: [],+ });const groupProfile = makeProfile({first_name: '',email: '',groups: ['group-abc'],});- // Sequential: identify first, then groupawait profileBuffer.add(identifyProfile);await profileBuffer.add(groupProfile);+ await profileBuffer.processBuffer();- // Second add should read the cached identify profile and merge groups inconst cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');expect(cached?.first_name).toBe('John');expect(cached?.email).toBe('[email protected]');expect(cached?.groups).toContain('group-abc');});- it('race condition: concurrent identify + group calls preserve all data', async () => {+ it('merges concurrent updates for the same profile at flush time', async () => {const identifyProfile = makeProfile({first_name: 'John',email: '[email protected]',groups: [],});-const groupProfile = makeProfile({first_name: '',email: '',groups: ['group-abc'],});- // Both calls run concurrently — the per-profile lock serializes them so the- // second one reads the first's result from cache and merges correctly.await Promise.all([profileBuffer.add(identifyProfile),profileBuffer.add(groupProfile),]);+ await profileBuffer.processBuffer();const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');-expect(cached?.first_name).toBe('John');expect(cached?.email).toBe('[email protected]');expect(cached?.groups).toContain('group-abc');});- it('race condition: concurrent writes produce one merged buffer entry', async () => {- const identifyProfile = makeProfile({- first_name: 'John',- email: '[email protected]',- groups: [],+ it('uses existing ClickHouse data for cache misses when merging', async () => {+ const existingInClickhouse = makeProfile({+ first_name: 'Jane',+ email: '[email protected]',+ groups: ['existing-group'],});+ vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);- const groupProfile = makeProfile({+ const incomingProfile = makeProfile({first_name: '',email: '',- groups: ['group-abc'],+ groups: ['new-group'],});- const sizeBefore = await profileBuffer.getBufferSize();+ await profileBuffer.add(incomingProfile);+ await profileBuffer.processBuffer();- await Promise.all([- profileBuffer.add(identifyProfile),- profileBuffer.add(groupProfile),- ]);+ const cached = await profileBuffer.fetchFromCache('profile-1', 'project-1');+ expect(cached?.first_name).toBe('Jane');+ expect(cached?.email).toBe('[email protected]');+ expect(cached?.groups).toContain('existing-group');+ expect(cached?.groups).toContain('new-group');+ });- const sizeAfter = await profileBuffer.getBufferSize();+ it('buffer is empty after flush', async () => {+ await profileBuffer.add(makeProfile({ first_name: 'John' }));+ expect(await profileBuffer.getBufferSize()).toBe(1);- // The second add merges into the first — only 2 buffer entries total- // (one from identify, one merged update with group)- expect(sizeAfter).toBe(sizeBefore + 2);+ await profileBuffer.processBuffer();++ expect(await profileBuffer.getBufferSize()).toBe(0);+ });++ it('retains profiles in queue when ClickHouse insert fails', async () => {+ await profileBuffer.add(makeProfile({ first_name: 'John' }));++ const { ch } = await import('../clickhouse/client');+ const insertSpy = vi+ .spyOn(ch, 'insert')+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));++ await profileBuffer.processBuffer();++ // Profiles must still be in the queue — not lost+ expect(await profileBuffer.getBufferSize()).toBe(1);++ insertSpy.mockRestore();+ });++ it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {+ vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));++ const { ch } = await import('../clickhouse/client');+ const insertSpy = vi+ .spyOn(ch, 'insert')+ .mockResolvedValueOnce(undefined as any);++ await profileBuffer.add(makeProfile({ first_name: 'John' }));+ await profileBuffer.processBuffer();- // The last entry in the buffer should have both name and group- const rawEntries = await redis.lrange('profile-buffer', 0, -1);- const entries = rawEntries.map((e) => getSafeJson<IClickhouseProfile>(e));- const lastEntry = entries[entries.length - 1];+ // Insert must still have been called — no data loss even when fetch fails+ expect(insertSpy).toHaveBeenCalled();+ expect(await profileBuffer.getBufferSize()).toBe(0);- expect(lastEntry?.first_name).toBe('John');- expect(lastEntry?.groups).toContain('group-abc');+ insertSpy.mockRestore();});});diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 6ff7c00be..e968951c7 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -1,9 +1,6 @@import { deepMergeObjects } from '@openpanel/common';-import { generateSecureId } from '@openpanel/common/server';import { getSafeJson } from '@openpanel/json';-import type { ILogger } from '@openpanel/logger';import { getRedisCache, type Redis } from '@openpanel/redis';-import shallowEqual from 'fast-deep-equal';import { omit, uniq } from 'ramda';import sqlstring from 'sqlstring';import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';@@ -11,29 +8,24 @@ import type { IClickhouseProfile } from '../services/profile.service';import { BaseBuffer } from './base-buffer';export class ProfileBuffer extends BaseBuffer {- private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE+ private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10): 200;- private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE+ private readonly chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10): 1000;- private ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS+ private readonly ttlInSeconds = process.env.PROFILE_BUFFER_TTL_IN_SECONDS? Number.parseInt(process.env.PROFILE_BUFFER_TTL_IN_SECONDS, 10): 60 * 60;+ /** Max profiles per ClickHouse IN-clause fetch to keep query size bounded */+ private readonly fetchChunkSize = process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE+ ? Number.parseInt(process.env.PROFILE_BUFFER_FETCH_CHUNK_SIZE, 10)+ : 50;private readonly redisKey = 'profile-buffer';private readonly redisProfilePrefix = 'profile-cache:';- private redis: Redis;- private releaseLockSha: string | null = null;-- private readonly releaseLockScript = `- if redis.call("get", KEYS[1]) == ARGV[1] then- return redis.call("del", KEYS[1])- else- return 0- end- `;+ private readonly redis: Redis;constructor() {super({@@ -43,9 +35,6 @@ export class ProfileBuffer extends BaseBuffer {},});this.redis = getRedisCache();- this.redis.script('LOAD', this.releaseLockScript).then((sha) => {- this.releaseLockSha = sha as string;- });}private getProfileCacheKey({@@ -58,243 +47,236 @@ export class ProfileBuffer extends BaseBuffer {return `${this.redisProfilePrefix}${projectId}:${profileId}`;}- private async withProfileLock<T>(+ public async fetchFromCache(profileId: string,- projectId: string,- fn: () => Promise<T>- ): Promise<T> {- const lockKey = `profile-lock:${projectId}:${profileId}`;- const lockId = generateSecureId('lock');- const maxRetries = 20;- const retryDelayMs = 50;-- for (let i = 0; i < maxRetries; i++) {- const acquired = await this.redis.set(lockKey, lockId, 'EX', 5, 'NX');- if (acquired === 'OK') {- try {- return await fn();- } finally {- if (this.releaseLockSha) {- await this.redis.evalsha(this.releaseLockSha, 1, lockKey, lockId);- } else {- await this.redis.eval(this.releaseLockScript, 1, lockKey, lockId);- }- }- }- await new Promise((resolve) => setTimeout(resolve, retryDelayMs));+ projectId: string+ ): Promise<IClickhouseProfile | null> {+ const cacheKey = this.getProfileCacheKey({ profileId, projectId });+ const cached = await this.redis.get(cacheKey);+ if (!cached) {+ return null;}-- this.logger.error(- 'Failed to acquire profile lock, proceeding without lock',- {- profileId,- projectId,- }- );- return fn();- }-- async alreadyExists(profile: IClickhouseProfile) {- const cacheKey = this.getProfileCacheKey({- profileId: profile.id,- projectId: profile.project_id,- });- return (await this.redis.exists(cacheKey)) === 1;+ return getSafeJson<IClickhouseProfile>(cached);}async add(profile: IClickhouseProfile, isFromEvent = false) {- const logger = this.logger.child({- projectId: profile.project_id,- profileId: profile.id,- });-try {- logger.debug('Adding profile');-- if (isFromEvent && (await this.alreadyExists(profile))) {- logger.debug('Profile already created, skipping');- return;- }-- await this.withProfileLock(profile.id, profile.project_id, async () => {- const existingProfile = await this.fetchProfile(profile, logger);-- // Delete any properties that are not server related if we have a non-server profile- if (- existingProfile?.properties.device !== 'server' &&- profile.properties.device === 'server'- ) {- profile.properties = omit(- [- 'city',- 'country',- 'region',- 'longitude',- 'latitude',- 'os',- 'osVersion',- 'browser',- 'device',- 'isServer',- 'os_version',- 'browser_version',- ],- profile.properties- );- }-- const mergedProfile: IClickhouseProfile = existingProfile- ? {- ...deepMergeObjects(- existingProfile,- omit(['created_at', 'groups'], profile)- ),- groups: uniq([- ...(existingProfile.groups ?? []),- ...(profile.groups ?? []),- ]),- }- : profile;-- if (- profile &&- existingProfile &&- shallowEqual(- omit(['created_at'], existingProfile),- omit(['created_at'], mergedProfile)- )- ) {- this.logger.debug('Profile not changed, skipping');- return;- }-- this.logger.debug('Merged profile will be inserted', {- mergedProfile,- existingProfile,- profile,- });-+ if (isFromEvent) {const cacheKey = this.getProfileCacheKey({profileId: profile.id,projectId: profile.project_id,});-- const result = await this.redis- .multi()- .set(cacheKey, JSON.stringify(mergedProfile), 'EX', this.ttlInSeconds)- .rpush(this.redisKey, JSON.stringify(mergedProfile))- .incr(this.bufferCounterKey)- .llen(this.redisKey)- .exec();-- if (!result) {- this.logger.error('Failed to add profile to Redis', {- profile,- cacheKey,- });+ const exists = await this.redis.exists(cacheKey);+ if (exists === 1) {return;}- const bufferLength = (result?.[3]?.[1] as number) ?? 0;+ }- this.logger.debug('Current buffer length', {- bufferLength,- batchSize: this.batchSize,- });- if (bufferLength >= this.batchSize) {- await this.tryFlush();- }- });+ const result = await this.redis+ .multi()+ .rpush(this.redisKey, JSON.stringify(profile))+ .incr(this.bufferCounterKey)+ .llen(this.redisKey)+ .exec();++ if (!result) {+ this.logger.error('Failed to add profile to Redis', { profile });+ return;+ }++ const bufferLength = (result?.[2]?.[1] as number) ?? 0;+ if (bufferLength >= this.batchSize) {+ await this.tryFlush();+ }} catch (error) {this.logger.error('Failed to add profile', { error, profile });}}- private async fetchProfile(- profile: IClickhouseProfile,- logger: ILogger- ): Promise<IClickhouseProfile | null> {- const existingProfile = await this.fetchFromCache(- profile.id,- profile.project_id- );- if (existingProfile) {- logger.debug('Profile found in Redis');- return existingProfile;+ private mergeProfiles(+ existing: IClickhouseProfile | null,+ incoming: IClickhouseProfile+ ): IClickhouseProfile {+ if (!existing) {+ return incoming;}- return this.fetchFromClickhouse(profile, logger);- }-- public async fetchFromCache(- profileId: string,- projectId: string- ): Promise<IClickhouseProfile | null> {- const cacheKey = this.getProfileCacheKey({- profileId,- projectId,- });- const existingProfile = await this.redis.get(cacheKey);- if (!existingProfile) {- return null;+ let profile = incoming;+ if (+ existing.properties.device !== 'server' &&+ incoming.properties.device === 'server'+ ) {+ profile = {+ ...incoming,+ properties: omit(+ [+ 'city',+ 'country',+ 'region',+ 'longitude',+ 'latitude',+ 'os',+ 'osVersion',+ 'browser',+ 'device',+ 'isServer',+ 'os_version',+ 'browser_version',+ ],+ incoming.properties+ ),+ };}- return getSafeJson<IClickhouseProfile>(existingProfile);++ return {+ ...deepMergeObjects(existing, omit(['created_at', 'groups'], profile)),+ groups: uniq([...(existing.groups ?? []), ...(incoming.groups ?? [])]),+ };}- private async fetchFromClickhouse(- profile: IClickhouseProfile,- logger: ILogger- ): Promise<IClickhouseProfile | null> {- logger.debug('Fetching profile from Clickhouse');- const result = await chQuery<IClickhouseProfile>(- `SELECT- id,- project_id,- last_value(nullIf(first_name, '')) as first_name,- last_value(nullIf(last_name, '')) as last_name,- last_value(nullIf(email, '')) as email,- last_value(nullIf(avatar, '')) as avatar,- last_value(is_external) as is_external,- last_value(properties) as properties,- last_value(created_at) as created_at- FROM ${TABLE_NAMES.profiles}- WHERE- id = ${sqlstring.escape(String(profile.id))} AND- project_id = ${sqlstring.escape(profile.project_id)}- ${- profile.is_external === false- ? ' AND profiles.created_at > now() - INTERVAL 2 DAY'- : ''+ private async batchFetchFromClickhouse(+ profiles: IClickhouseProfile[]+ ): Promise<Map<string, IClickhouseProfile>> {+ const result = new Map<string, IClickhouseProfile>();++ // Non-external (anonymous/device) profiles get a 2-day recency filter to+ // avoid pulling stale anonymous sessions from far back.+ const external = profiles.filter((p) => p.is_external !== false);+ const nonExternal = profiles.filter((p) => p.is_external === false);++ const fetchGroup = async (+ group: IClickhouseProfile[],+ withDateFilter: boolean+ ) => {+ for (const chunk of this.chunks(group, this.fetchChunkSize)) {+ const tuples = chunk+ .map(+ (p) =>+ `(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})`+ )+ .join(', ');+ try {+ const rows = await chQuery<IClickhouseProfile>(+ `SELECT+ id,+ project_id,+ argMax(nullIf(first_name, ''), ${TABLE_NAMES.profiles}.created_at) as first_name,+ argMax(nullIf(last_name, ''), ${TABLE_NAMES.profiles}.created_at) as last_name,+ argMax(nullIf(email, ''), ${TABLE_NAMES.profiles}.created_at) as email,+ argMax(nullIf(avatar, ''), ${TABLE_NAMES.profiles}.created_at) as avatar,+ argMax(is_external, ${TABLE_NAMES.profiles}.created_at) as is_external,+ argMax(properties, ${TABLE_NAMES.profiles}.created_at) as properties,+ max(created_at) as created_at+ FROM ${TABLE_NAMES.profiles}+ WHERE (id, project_id) IN (${tuples})+ ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.created_at > now() - INTERVAL 2 DAY` : ''}+ GROUP BY id, project_id`+ );+ for (const row of rows) {+ result.set(`${row.project_id}:${row.id}`, row);+ }+ } catch (error) {+ this.logger.warn(+ 'Failed to batch fetch profiles from Clickhouse, proceeding without existing data',+ { error, chunkSize: chunk.length }+ );}- GROUP BY id, project_id- ORDER BY created_at DESC- LIMIT 1`- );- logger.debug('Clickhouse fetch result', {- found: !!result[0],- });- return result[0] || null;+ }+ };++ await Promise.all([+ fetchGroup(external, false),+ fetchGroup(nonExternal, true),+ ]);++ return result;}async processBuffer() {try {this.logger.debug('Starting profile buffer processing');- const profiles = await this.redis.lrange(+ const rawProfiles = await this.redis.lrange(this.redisKey,0,this.batchSize - 1);- if (profiles.length === 0) {+ if (rawProfiles.length === 0) {this.logger.debug('No profiles to process');return;}- this.logger.debug(`Processing ${profiles.length} profiles in buffer`);- const parsedProfiles = profiles.map((p) =>- getSafeJson<IClickhouseProfile>(p)+ const parsedProfiles = rawProfiles+ .map((p) => getSafeJson<IClickhouseProfile>(p))+ .filter(Boolean) as IClickhouseProfile[];++ // Merge within batch: collapse multiple updates for the same profile+ const mergedInBatch = new Map<string, IClickhouseProfile>();+ for (const profile of parsedProfiles) {+ const key = `${profile.project_id}:${profile.id}`;+ mergedInBatch.set(+ key,+ this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)+ );+ }++ const uniqueProfiles = Array.from(mergedInBatch.values());++ // Check Redis cache for all unique profiles in a single MGET+ const cacheKeys = uniqueProfiles.map((p) =>+ this.getProfileCacheKey({ profileId: p.id, projectId: p.project_id }));+ const cacheResults = await this.redis.mget(...cacheKeys);++ const existingByKey = new Map<string, IClickhouseProfile>();+ const cacheMisses: IClickhouseProfile[] = [];+ for (let i = 0; i < uniqueProfiles.length; i++) {+ const uniqueProfile = uniqueProfiles[i];+ if (uniqueProfile) {+ const key = `${uniqueProfile.project_id}:${uniqueProfile.id}`;+ const cached = cacheResults[i]+ ? getSafeJson<IClickhouseProfile>(cacheResults[i]!)+ : null;+ if (cached) {+ existingByKey.set(key, cached);+ } else {+ cacheMisses.push(uniqueProfile);+ }+ }+ }++ // Fetch cache misses from ClickHouse in bounded chunks+ if (cacheMisses.length > 0) {+ const clickhouseResults =+ await this.batchFetchFromClickhouse(cacheMisses);+ for (const [key, profile] of clickhouseResults) {+ existingByKey.set(key, profile);+ }+ }- for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {+ // Final merge: in-batch profile + existing (from cache or ClickHouse)+ const toInsert: IClickhouseProfile[] = [];+ const multi = this.redis.multi();++ for (const profile of uniqueProfiles) {+ const key = `${profile.project_id}:${profile.id}`;+ const merged = this.mergeProfiles(+ existingByKey.get(key) ?? null,+ profile+ );+ toInsert.push(merged);+ multi.set(+ this.getProfileCacheKey({+ projectId: profile.project_id,+ profileId: profile.id,+ }),+ JSON.stringify(merged),+ 'EX',+ this.ttlInSeconds+ );+ }++ for (const chunk of this.chunks(toInsert, this.chunkSize)) {await ch.insert({table: TABLE_NAMES.profiles,values: chunk,@@ -302,22 +284,21 @@ export class ProfileBuffer extends BaseBuffer {});}- // Only remove profiles after successful insert and update counter- await this.redis- .multi()- .ltrim(this.redisKey, profiles.length, -1)- .decrby(this.bufferCounterKey, profiles.length)- .exec();+ multi+ .ltrim(this.redisKey, rawProfiles.length, -1)+ .decrby(this.bufferCounterKey, rawProfiles.length);+ await multi.exec();this.logger.debug('Successfully completed profile processing', {- totalProfiles: profiles.length,+ totalProfiles: rawProfiles.length,+ uniqueProfiles: uniqueProfiles.length,});} catch (error) {this.logger.error('Failed to process buffer', { error });}}- async getBufferSize() {+ getBufferSize() {return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));}}diff --git a/packages/db/src/buffers/session-buffer.test.ts b/packages/db/src/buffers/session-buffer.test.tsnew file mode 100644index 000000000..f140ff002--- /dev/null+++ b/packages/db/src/buffers/session-buffer.test.ts@@ -0,0 +1,122 @@+import { getRedisCache } from '@openpanel/redis';+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { ch } from '../clickhouse/client';++vi.mock('../clickhouse/client', () => ({+ ch: {+ insert: vi.fn().mockResolvedValue(undefined),+ },+ TABLE_NAMES: {+ sessions: 'sessions',+ },+}));++import { SessionBuffer } from './session-buffer';+import type { IClickhouseEvent } from '../services/event.service';++const redis = getRedisCache();++function makeEvent(overrides: Partial<IClickhouseEvent>): IClickhouseEvent {+ return {+ id: 'event-1',+ project_id: 'project-1',+ profile_id: 'profile-1',+ device_id: 'device-1',+ session_id: 'session-1',+ name: 'screen_view',+ path: '/home',+ origin: '',+ referrer: '',+ referrer_name: '',+ referrer_type: '',+ duration: 0,+ properties: {},+ created_at: new Date().toISOString(),+ groups: [],+ ...overrides,+ } as IClickhouseEvent;+}++beforeEach(async () => {+ const keys = [+ ...await redis.keys('session*'),+ ...await redis.keys('lock:session'),+ ];+ if (keys.length > 0) await redis.del(...keys);+ vi.mocked(ch.insert).mockResolvedValue(undefined as any);+});++afterAll(async () => {+ try {+ await redis.quit();+ } catch {}+});++describe('SessionBuffer', () => {+ let sessionBuffer: SessionBuffer;++ beforeEach(() => {+ sessionBuffer = new SessionBuffer();+ });++ it('adds a new session to the buffer', async () => {+ const sizeBefore = await sessionBuffer.getBufferSize();+ await sessionBuffer.add(makeEvent({}));+ const sizeAfter = await sessionBuffer.getBufferSize();++ expect(sizeAfter).toBe(sizeBefore + 1);+ });++ it('skips session_start and session_end events', async () => {+ const sizeBefore = await sessionBuffer.getBufferSize();+ await sessionBuffer.add(makeEvent({ name: 'session_start' }));+ await sessionBuffer.add(makeEvent({ name: 'session_end' }));+ const sizeAfter = await sessionBuffer.getBufferSize();++ expect(sizeAfter).toBe(sizeBefore);+ });++ it('updates existing session on subsequent events', async () => {+ const t0 = Date.now();+ await sessionBuffer.add(makeEvent({ created_at: new Date(t0).toISOString() }));++ // Second event updates the same session — emits old (sign=-1) + new (sign=1)+ const sizeBefore = await sessionBuffer.getBufferSize();+ await sessionBuffer.add(makeEvent({ created_at: new Date(t0 + 5000).toISOString() }));+ const sizeAfter = await sessionBuffer.getBufferSize();++ expect(sizeAfter).toBe(sizeBefore + 2);+ });++ it('processes buffer and inserts sessions into ClickHouse', async () => {+ await sessionBuffer.add(makeEvent({}));++ const insertSpy = vi+ .spyOn(ch, 'insert')+ .mockResolvedValueOnce(undefined as any);++ await sessionBuffer.processBuffer();++ expect(insertSpy).toHaveBeenCalledWith(+ expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })+ );+ expect(await sessionBuffer.getBufferSize()).toBe(0);++ insertSpy.mockRestore();+ });++ it('retains sessions in queue when ClickHouse insert fails', async () => {+ await sessionBuffer.add(makeEvent({}));++ const insertSpy = vi+ .spyOn(ch, 'insert')+ .mockRejectedValueOnce(new Error('ClickHouse unavailable'));++ await sessionBuffer.processBuffer();++ // Sessions must still be in the queue — not lost+ expect(await sessionBuffer.getBufferSize()).toBe(1);++ insertSpy.mockRestore();+ });+});diff --git a/packages/redis/cachable.test.ts b/packages/redis/cachable.test.tsindex d628f4391..2d212a2ea 100644--- a/packages/redis/cachable.test.ts+++ b/packages/redis/cachable.test.ts@@ -8,7 +8,10 @@ describe('cachable', () => {beforeEach(async () => {redis = getRedisCache();// Clear any existing cache data for clean tests- const keys = await redis.keys('cachable:*');+ const keys = [+ ...await redis.keys('cachable:*'),+ ...await redis.keys('test-key*'),+ ];if (keys.length > 0) {await redis.del(...keys);}@@ -16,7 +19,10 @@ describe('cachable', () => {afterEach(async () => {// Clean up after each test- const keys = await redis.keys('cachable:*');+ const keys = [+ ...await redis.keys('cachable:*'),+ ...await redis.keys('test-key*'),+ ];if (keys.length > 0) {await redis.del(...keys);}