Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Handle fragmented replay events in recording and playback

Mike Cao15038b09d9fa
diff --git a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx
index b4c455cb0..95ca3e883 100644
--- a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx
+++ b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx
@@ -1,7 +1,9 @@
'use client';
import { Column } from '@umami/react-zen';
import { useEffect, useRef, useState } from 'react';
+import { Empty } from '@/components/common/Empty';
import { useMobile } from '@/components/hooks';
+import { hasReplayFullSnapshot } from '@/lib/replay';
import 'rrweb-player/dist/style.css';
export function ReplayPlayer({ events }: { events: any[] }) {
@@ -9,12 +11,14 @@ export function ReplayPlayer({ events }: { events: any[] }) {
const playerRef = useRef<any>(null);
const [loaded, setLoaded] = useState(false);
const { isMobile, isPhone } = useMobile();
+ const canReplay = hasReplayFullSnapshot(events);
+ const showUnavailable = !events?.length || !canReplay;
const playerWidth = isPhone ? 360 : isMobile ? 640 : 1024;
const playerHeight = isPhone ? 202 : isMobile ? 360 : 576;
useEffect(() => {
- if (!containerRef.current || !events?.length) return;
+ if (!containerRef.current || !events?.length || !canReplay) return;
import('rrweb-player').then(mod => {
const RRWebPlayer = mod.default;
@@ -46,7 +50,7 @@ export function ReplayPlayer({ events }: { events: any[] }) {
playerRef.current = null;
}
};
- }, [events, playerWidth, playerHeight]);
+ }, [canReplay, events, playerWidth, playerHeight]);
return (
<Column alignItems="center">
@@ -54,14 +58,16 @@ export function ReplayPlayer({ events }: { events: any[] }) {
ref={containerRef}
style={{
width: playerWidth,
- minHeight: loaded ? undefined : playerHeight,
+ minHeight: loaded && canReplay ? undefined : playerHeight,
maxWidth: '100%',
overflow: 'hidden',
borderRadius: '8px',
border: '1px solid var(--base300)',
background: 'var(--base75)',
}}
- />
+ >
+ {showUnavailable && <Empty message="Replay unavailable." />}
+ </div>
</Column>
);
}
diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.ts
index 7fa267b79..ade6aa0c2 100644
--- a/src/app/api/record/route.ts
+++ b/src/app/api/record/route.ts
@@ -7,6 +7,7 @@ import { getClientInfo, hasBlockedIp } from '@/lib/detect';
import { parseToken } from '@/lib/jwt';
import { fetchAccount, fetchTeam } from '@/lib/load';
import { getRecorderConfig } from '@/lib/recorder';
+import { getReplayEventCount } from '@/lib/replay';
import { parseRequest } from '@/lib/request';
import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response';
import { getWebsite } from '@/queries/prisma';
@@ -197,7 +198,7 @@ export async function POST(request: Request) {
visitId,
chunkIndex,
events,
- eventCount: events.length,
+ eventCount: getReplayEventCount(events),
startedAt,
endedAt,
});
diff --git a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts b/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts
index a35afae55..1720a4501 100644
--- a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts
+++ b/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts
@@ -1,3 +1,4 @@
+import { restoreReplayEventFragments } from '@/lib/replay';
import { parseRequest } from '@/lib/request';
import { json, unauthorized } from '@/lib/response';
import { canViewAuthenticatedWebsite } from '@/permissions';
@@ -91,7 +92,9 @@ export async function GET(
}
const chunks = await getReplayChunks(websiteId, replayId, { endAt, endChunkIndex });
- const allEvents = mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex });
+ const allEvents = restoreReplayEventFragments(
+ mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex }),
+ );
const sessionId = chunks.length > 0 ? chunks[0].sessionId : null;
const startedAt = chunks.length > 0 ? chunks[0].startedAt : null;
const endedAt = chunks.length > 0 ? chunks[chunks.length - 1].endedAt : null;
diff --git a/src/lib/replay.test.ts b/src/lib/replay.test.ts
new file mode 100644
index 000000000..d875b10e5
--- /dev/null
+++ b/src/lib/replay.test.ts
@@ -0,0 +1,84 @@
+import { expect, test } from 'vitest';
+import {
+ getReplayEventCount,
+ hasReplayFullSnapshot,
+ REPLAY_EVENT_FRAGMENT_TYPE,
+ restoreReplayEventFragments,
+} from './replay';
+
+test('hasReplayFullSnapshot returns true when a full snapshot is present', () => {
+ expect(hasReplayFullSnapshot([{ type: 4 }, { type: 2 }, { type: 3 }])).toBe(true);
+});
+
+test('hasReplayFullSnapshot returns false when only incremental events are present', () => {
+ expect(hasReplayFullSnapshot([{ type: 4 }, { type: 3 }, { type: 3 }])).toBe(false);
+});
+
+test('hasReplayFullSnapshot handles missing events', () => {
+ expect(hasReplayFullSnapshot(null)).toBe(false);
+ expect(hasReplayFullSnapshot(undefined)).toBe(false);
+});
+
+test('restoreReplayEventFragments restores fragmented events', () => {
+ const fullSnapshot = {
+ type: 2,
+ timestamp: 1781553116151,
+ data: {
+ node: {
+ type: 0,
+ childNodes: [{ type: 2, tagName: 'html' }],
+ },
+ },
+ };
+ const serialized = JSON.stringify(fullSnapshot);
+ const splitAt = Math.floor(serialized.length / 2);
+
+ const events = restoreReplayEventFragments([
+ { type: 4, timestamp: 1781553116150 },
+ {
+ type: REPLAY_EVENT_FRAGMENT_TYPE,
+ timestamp: fullSnapshot.timestamp,
+ data: {
+ id: 'snapshot-1',
+ index: 0,
+ total: 2,
+ value: serialized.slice(0, splitAt),
+ },
+ },
+ {
+ type: REPLAY_EVENT_FRAGMENT_TYPE,
+ timestamp: fullSnapshot.timestamp,
+ data: {
+ id: 'snapshot-1',
+ index: 1,
+ total: 2,
+ value: serialized.slice(splitAt),
+ },
+ },
+ { type: 3, timestamp: 1781553116160 },
+ ]);
+
+ expect(events).toEqual([
+ { type: 4, timestamp: 1781553116150 },
+ fullSnapshot,
+ { type: 3, timestamp: 1781553116160 },
+ ]);
+ expect(hasReplayFullSnapshot(events)).toBe(true);
+});
+
+test('getReplayEventCount counts a fragment group as one event', () => {
+ expect(
+ getReplayEventCount([
+ { type: 4 },
+ {
+ type: REPLAY_EVENT_FRAGMENT_TYPE,
+ data: { id: 'snapshot-1', index: 0, total: 2, value: '{' },
+ },
+ {
+ type: REPLAY_EVENT_FRAGMENT_TYPE,
+ data: { id: 'snapshot-1', index: 1, total: 2, value: '}' },
+ },
+ { type: 3 },
+ ]),
+ ).toBe(3);
+});
diff --git a/src/lib/replay.ts b/src/lib/replay.ts
new file mode 100644
index 000000000..0ad8c96ba
--- /dev/null
+++ b/src/lib/replay.ts
@@ -0,0 +1,108 @@
+export const RRWEB_EVENT_TYPE = {
+ FullSnapshot: 2,
+} as const;
+
+export const REPLAY_EVENT_FRAGMENT_TYPE = 'umami:rrweb-event-fragment';
+
+interface ReplayEventFragment {
+ type: typeof REPLAY_EVENT_FRAGMENT_TYPE;
+ timestamp?: number;
+ data: {
+ id: string;
+ index: number;
+ total: number;
+ value: string;
+ };
+}
+
+export function isReplayEventFragment(event: any): event is ReplayEventFragment {
+ const { data } = event || {};
+
+ return (
+ event?.type === REPLAY_EVENT_FRAGMENT_TYPE &&
+ typeof data?.id === 'string' &&
+ Number.isInteger(data?.index) &&
+ Number.isInteger(data?.total) &&
+ data.index >= 0 &&
+ data.index < data.total &&
+ typeof data?.value === 'string'
+ );
+}
+
+export function getReplayEventCount(events: any[] | null | undefined) {
+ if (!Array.isArray(events)) {
+ return 0;
+ }
+
+ return events.reduce((count, event) => {
+ if (isReplayEventFragment(event)) {
+ return count + (event.data.index === 0 ? 1 : 0);
+ }
+
+ return count + 1;
+ }, 0);
+}
+
+export function restoreReplayEventFragments(events: any[] | null | undefined) {
+ if (!Array.isArray(events)) {
+ return [];
+ }
+
+ const restored: any[] = [];
+ const pending = new Map<
+ string,
+ {
+ total: number;
+ values: Map<number, string>;
+ received: number;
+ }
+ >();
+
+ for (const event of events) {
+ if (!isReplayEventFragment(event)) {
+ restored.push(event);
+ continue;
+ }
+
+ const { id, index, total, value } = event.data;
+ let fragment = pending.get(id);
+
+ if (!fragment || fragment.total !== total) {
+ fragment = {
+ total,
+ values: new Map(),
+ received: 0,
+ };
+ pending.set(id, fragment);
+ }
+
+ if (!fragment.values.has(index)) {
+ fragment.values.set(index, value);
+ fragment.received += 1;
+ }
+
+ if (fragment.received === fragment.total) {
+ pending.delete(id);
+
+ try {
+ let serialized = '';
+
+ for (let i = 0; i < fragment.total; i++) {
+ serialized += fragment.values.get(i) || '';
+ }
+
+ restored.push(JSON.parse(serialized));
+ } catch {
+ // Ignore malformed fragment groups. A partial replay is better than failing the whole response.
+ }
+ }
+ }
+
+ return restored;
+}
+
+export function hasReplayFullSnapshot(events: any[] | null | undefined) {
+ return (
+ Array.isArray(events) && events.some(event => event?.type === RRWEB_EVENT_TYPE.FullSnapshot)
+ );
+}
diff --git a/src/recorder/index.js b/src/recorder/index.js
index 9f5950ec3..1d36341cb 100644
--- a/src/recorder/index.js
+++ b/src/recorder/index.js
@@ -23,7 +23,9 @@ import { record } from 'rrweb';
const REPLAY_FLUSH_EVENT_COUNT = 100;
const REPLAY_FLUSH_INTERVAL = 2000;
- const REPLAY_MAX_PAYLOAD_SIZE = 900000;
+ const REPLAY_MAX_PAYLOAD_SIZE = 950000;
+ const REPLAY_FRAGMENT_TYPE = 'umami:rrweb-event-fragment';
+ const REPLAY_FRAGMENT_TOTAL_PLACEHOLDER = 999999999;
const HEATMAP_FLUSH_EVENT_COUNT = 20;
const HEATMAP_FLUSH_INTERVAL = 5000;
@@ -41,6 +43,7 @@ import { record } from 'rrweb';
let replayFlushTimer = null;
let heatmapFlushTimer = null;
let replayStartTime = null;
+ let replayLastChunkIndex = 0;
let replayStopped = false;
let heatmapStarted = false;
@@ -69,6 +72,35 @@ import { record } from 'rrweb';
const isReplayPayloadTooLarge = (events, timestamp) =>
getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE;
+ const getReplayChunkIndex = () => {
+ const timestamp = Math.floor(Date.now() / 1000);
+ const chunkIndex = Math.max(timestamp, replayLastChunkIndex + 1);
+
+ replayLastChunkIndex = chunkIndex;
+
+ return chunkIndex;
+ };
+
+ const createReplayFragment = (id, index, total, timestamp, value) => ({
+ type: REPLAY_FRAGMENT_TYPE,
+ timestamp,
+ data: {
+ id,
+ index,
+ total,
+ value,
+ },
+ });
+
+ const getReplayEventTimestamp = event => {
+ const timestamp = Number(event?.timestamp);
+
+ return Number.isFinite(timestamp) ? timestamp : Date.now();
+ };
+
+ const getReplayFragmentId = () =>
+ `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+
const sendPayload = (type, payload, useKeepalive = false) => {
const cache = getSessionCache();
@@ -90,45 +122,106 @@ import { record } from 'rrweb';
}).catch(() => {});
};
+ const sendReplayChunk = (events, timestamp, useKeepalive = false) => {
+ replayLastChunkIndex = Math.max(replayLastChunkIndex, timestamp);
+
+ sendPayload(
+ 'record',
+ {
+ events,
+ timestamp,
+ },
+ useKeepalive,
+ );
+ };
+
+ const getReplayEventFragments = (event, chunkTimestamp) => {
+ const value = JSON.stringify(event);
+ const id = getReplayFragmentId();
+ const eventTimestamp = getReplayEventTimestamp(event);
+ const fragments = [];
+ let start = 0;
+
+ while (start < value.length) {
+ let low = start + 1;
+ let high = value.length;
+ let end = start;
+
+ while (low <= high) {
+ const mid = Math.floor((low + high) / 2);
+ const fragmentChunkTimestamp = chunkTimestamp + fragments.length;
+ const fragment = createReplayFragment(
+ id,
+ fragments.length,
+ REPLAY_FRAGMENT_TOTAL_PLACEHOLDER,
+ eventTimestamp,
+ value.slice(start, mid),
+ );
+
+ if (isReplayPayloadTooLarge([fragment], fragmentChunkTimestamp)) {
+ high = mid - 1;
+ } else {
+ end = mid;
+ low = mid + 1;
+ }
+ }
+
+ if (end === start) {
+ end = start + 1;
+ }
+
+ fragments.push(value.slice(start, end));
+ start = end;
+ }
+
+ return fragments.map((fragment, index) =>
+ createReplayFragment(id, index, fragments.length, eventTimestamp, fragment),
+ );
+ };
+
+ const sendReplayEventFragments = (event, timestamp, useKeepalive = false) => {
+ const fragments = getReplayEventFragments(event, timestamp);
+
+ fragments.forEach((fragment, index) => {
+ sendReplayChunk([fragment], timestamp + index, useKeepalive);
+ });
+
+ return fragments.length;
+ };
+
const sendReplayEvents = (events, timestamp, useKeepalive = false) => {
let chunk = [];
let chunkOffset = 0;
- events.forEach(event => {
- const candidate = [...chunk, event];
+ for (const event of events) {
+ const chunkTimestamp = timestamp + chunkOffset;
- if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) {
+ if (isReplayPayloadTooLarge([event], chunkTimestamp)) {
if (chunk.length) {
- sendPayload(
- 'record',
- {
- events: chunk,
- timestamp: timestamp + chunkOffset,
- },
- useKeepalive,
- );
+ sendReplayChunk(chunk, chunkTimestamp, useKeepalive);
chunk = [];
chunkOffset += 1;
}
- if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) {
+ chunkOffset += sendReplayEventFragments(event, timestamp + chunkOffset, useKeepalive);
+ continue;
+ }
+
+ const candidate = [...chunk, event];
+
+ if (isReplayPayloadTooLarge(candidate, chunkTimestamp)) {
+ if (chunk.length) {
+ sendReplayChunk(chunk, chunkTimestamp, useKeepalive);
+ chunk = [];
chunkOffset += 1;
- return;
}
}
chunk.push(event);
- });
+ }
if (chunk.length) {
- sendPayload(
- 'record',
- {
- events: chunk,
- timestamp: timestamp + chunkOffset,
- },
- useKeepalive,
- );
+ sendReplayChunk(chunk, timestamp + chunkOffset, useKeepalive);
}
};
@@ -138,7 +231,7 @@ import { record } from 'rrweb';
const events = replayBuffer;
replayBuffer = [];
- sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive);
+ sendReplayEvents(events, getReplayChunkIndex(), useKeepalive);
};
const flushHeatmap = (useKeepalive = false) => {
@@ -326,10 +419,21 @@ import { record } from 'rrweb';
return;
}
- if (
- replayBuffer.length &&
- isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000))
- ) {
+ const timestamp = Math.floor(Date.now() / 1000);
+
+ if (isReplayPayloadTooLarge([event], timestamp)) {
+ if (replayBuffer.length) {
+ const events = replayBuffer;
+ replayBuffer = [];
+
+ sendReplayEvents(events, getReplayChunkIndex());
+ }
+
+ sendReplayEvents([event], getReplayChunkIndex());
+ return;
+ }
+
+ if (replayBuffer.length && isReplayPayloadTooLarge([...replayBuffer, event], timestamp)) {
flushReplay();
}
@@ -359,6 +463,11 @@ import { record } from 'rrweb';
checkoutEveryNms: 30000,
...(blockSelector && { blockSelector }),
});
+
+ if (replayStopped && replayStopFn) {
+ replayStopFn();
+ replayStopFn = null;
+ }
};
const beginHeatmapCapture = () => {