Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Handle fragmented replay events in recording and playback
Mike Cao15038b09d9fa
diff --git a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsxindex b4c455cb0..95ca3e883 100644--- a/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx+++ b/src/app/(main)/websites/[websiteId]/replays/[replayId]/ReplayPlayer.tsx@@ -1,7 +1,9 @@'use client';import { Column } from '@umami/react-zen';import { useEffect, useRef, useState } from 'react';+import { Empty } from '@/components/common/Empty';import { useMobile } from '@/components/hooks';+import { hasReplayFullSnapshot } from '@/lib/replay';import 'rrweb-player/dist/style.css';export function ReplayPlayer({ events }: { events: any[] }) {@@ -9,12 +11,14 @@ export function ReplayPlayer({ events }: { events: any[] }) {const playerRef = useRef<any>(null);const [loaded, setLoaded] = useState(false);const { isMobile, isPhone } = useMobile();+ const canReplay = hasReplayFullSnapshot(events);+ const showUnavailable = !events?.length || !canReplay;const playerWidth = isPhone ? 360 : isMobile ? 640 : 1024;const playerHeight = isPhone ? 202 : isMobile ? 360 : 576;useEffect(() => {- if (!containerRef.current || !events?.length) return;+ if (!containerRef.current || !events?.length || !canReplay) return;import('rrweb-player').then(mod => {const RRWebPlayer = mod.default;@@ -46,7 +50,7 @@ export function ReplayPlayer({ events }: { events: any[] }) {playerRef.current = null;}};- }, [events, playerWidth, playerHeight]);+ }, [canReplay, events, playerWidth, playerHeight]);return (<Column alignItems="center">@@ -54,14 +58,16 @@ export function ReplayPlayer({ events }: { events: any[] }) {ref={containerRef}style={{width: playerWidth,- minHeight: loaded ? undefined : playerHeight,+ minHeight: loaded && canReplay ? undefined : playerHeight,maxWidth: '100%',overflow: 'hidden',borderRadius: '8px',border: '1px solid var(--base300)',background: 'var(--base75)',}}- />+ >+ {showUnavailable && <Empty message="Replay unavailable." />}+ </div></Column>);}diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.tsindex 7fa267b79..ade6aa0c2 100644--- a/src/app/api/record/route.ts+++ b/src/app/api/record/route.ts@@ -7,6 +7,7 @@ import { getClientInfo, hasBlockedIp } from '@/lib/detect';import { parseToken } from '@/lib/jwt';import { fetchAccount, fetchTeam } from '@/lib/load';import { getRecorderConfig } from '@/lib/recorder';+import { getReplayEventCount } from '@/lib/replay';import { parseRequest } from '@/lib/request';import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response';import { getWebsite } from '@/queries/prisma';@@ -197,7 +198,7 @@ export async function POST(request: Request) {visitId,chunkIndex,events,- eventCount: events.length,+ eventCount: getReplayEventCount(events),startedAt,endedAt,});diff --git a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts b/src/app/api/websites/[websiteId]/replays/[replayId]/route.tsindex a35afae55..1720a4501 100644--- a/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts+++ b/src/app/api/websites/[websiteId]/replays/[replayId]/route.ts@@ -1,3 +1,4 @@+import { restoreReplayEventFragments } from '@/lib/replay';import { parseRequest } from '@/lib/request';import { json, unauthorized } from '@/lib/response';import { canViewAuthenticatedWebsite } from '@/permissions';@@ -91,7 +92,9 @@ export async function GET(}const chunks = await getReplayChunks(websiteId, replayId, { endAt, endChunkIndex });- const allEvents = mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex });+ const allEvents = restoreReplayEventFragments(+ mergeReplayEvents(chunks, { until, endChunkIndex, endEventIndex }),+ );const sessionId = chunks.length > 0 ? chunks[0].sessionId : null;const startedAt = chunks.length > 0 ? chunks[0].startedAt : null;const endedAt = chunks.length > 0 ? chunks[chunks.length - 1].endedAt : null;diff --git a/src/lib/replay.test.ts b/src/lib/replay.test.tsnew file mode 100644index 000000000..d875b10e5--- /dev/null+++ b/src/lib/replay.test.ts@@ -0,0 +1,84 @@+import { expect, test } from 'vitest';+import {+ getReplayEventCount,+ hasReplayFullSnapshot,+ REPLAY_EVENT_FRAGMENT_TYPE,+ restoreReplayEventFragments,+} from './replay';++test('hasReplayFullSnapshot returns true when a full snapshot is present', () => {+ expect(hasReplayFullSnapshot([{ type: 4 }, { type: 2 }, { type: 3 }])).toBe(true);+});++test('hasReplayFullSnapshot returns false when only incremental events are present', () => {+ expect(hasReplayFullSnapshot([{ type: 4 }, { type: 3 }, { type: 3 }])).toBe(false);+});++test('hasReplayFullSnapshot handles missing events', () => {+ expect(hasReplayFullSnapshot(null)).toBe(false);+ expect(hasReplayFullSnapshot(undefined)).toBe(false);+});++test('restoreReplayEventFragments restores fragmented events', () => {+ const fullSnapshot = {+ type: 2,+ timestamp: 1781553116151,+ data: {+ node: {+ type: 0,+ childNodes: [{ type: 2, tagName: 'html' }],+ },+ },+ };+ const serialized = JSON.stringify(fullSnapshot);+ const splitAt = Math.floor(serialized.length / 2);++ const events = restoreReplayEventFragments([+ { type: 4, timestamp: 1781553116150 },+ {+ type: REPLAY_EVENT_FRAGMENT_TYPE,+ timestamp: fullSnapshot.timestamp,+ data: {+ id: 'snapshot-1',+ index: 0,+ total: 2,+ value: serialized.slice(0, splitAt),+ },+ },+ {+ type: REPLAY_EVENT_FRAGMENT_TYPE,+ timestamp: fullSnapshot.timestamp,+ data: {+ id: 'snapshot-1',+ index: 1,+ total: 2,+ value: serialized.slice(splitAt),+ },+ },+ { type: 3, timestamp: 1781553116160 },+ ]);++ expect(events).toEqual([+ { type: 4, timestamp: 1781553116150 },+ fullSnapshot,+ { type: 3, timestamp: 1781553116160 },+ ]);+ expect(hasReplayFullSnapshot(events)).toBe(true);+});++test('getReplayEventCount counts a fragment group as one event', () => {+ expect(+ getReplayEventCount([+ { type: 4 },+ {+ type: REPLAY_EVENT_FRAGMENT_TYPE,+ data: { id: 'snapshot-1', index: 0, total: 2, value: '{' },+ },+ {+ type: REPLAY_EVENT_FRAGMENT_TYPE,+ data: { id: 'snapshot-1', index: 1, total: 2, value: '}' },+ },+ { type: 3 },+ ]),+ ).toBe(3);+});diff --git a/src/lib/replay.ts b/src/lib/replay.tsnew file mode 100644index 000000000..0ad8c96ba--- /dev/null+++ b/src/lib/replay.ts@@ -0,0 +1,108 @@+export const RRWEB_EVENT_TYPE = {+ FullSnapshot: 2,+} as const;++export const REPLAY_EVENT_FRAGMENT_TYPE = 'umami:rrweb-event-fragment';++interface ReplayEventFragment {+ type: typeof REPLAY_EVENT_FRAGMENT_TYPE;+ timestamp?: number;+ data: {+ id: string;+ index: number;+ total: number;+ value: string;+ };+}++export function isReplayEventFragment(event: any): event is ReplayEventFragment {+ const { data } = event || {};++ return (+ event?.type === REPLAY_EVENT_FRAGMENT_TYPE &&+ typeof data?.id === 'string' &&+ Number.isInteger(data?.index) &&+ Number.isInteger(data?.total) &&+ data.index >= 0 &&+ data.index < data.total &&+ typeof data?.value === 'string'+ );+}++export function getReplayEventCount(events: any[] | null | undefined) {+ if (!Array.isArray(events)) {+ return 0;+ }++ return events.reduce((count, event) => {+ if (isReplayEventFragment(event)) {+ return count + (event.data.index === 0 ? 1 : 0);+ }++ return count + 1;+ }, 0);+}++export function restoreReplayEventFragments(events: any[] | null | undefined) {+ if (!Array.isArray(events)) {+ return [];+ }++ const restored: any[] = [];+ const pending = new Map<+ string,+ {+ total: number;+ values: Map<number, string>;+ received: number;+ }+ >();++ for (const event of events) {+ if (!isReplayEventFragment(event)) {+ restored.push(event);+ continue;+ }++ const { id, index, total, value } = event.data;+ let fragment = pending.get(id);++ if (!fragment || fragment.total !== total) {+ fragment = {+ total,+ values: new Map(),+ received: 0,+ };+ pending.set(id, fragment);+ }++ if (!fragment.values.has(index)) {+ fragment.values.set(index, value);+ fragment.received += 1;+ }++ if (fragment.received === fragment.total) {+ pending.delete(id);++ try {+ let serialized = '';++ for (let i = 0; i < fragment.total; i++) {+ serialized += fragment.values.get(i) || '';+ }++ restored.push(JSON.parse(serialized));+ } catch {+ // Ignore malformed fragment groups. A partial replay is better than failing the whole response.+ }+ }+ }++ return restored;+}++export function hasReplayFullSnapshot(events: any[] | null | undefined) {+ return (+ Array.isArray(events) && events.some(event => event?.type === RRWEB_EVENT_TYPE.FullSnapshot)+ );+}diff --git a/src/recorder/index.js b/src/recorder/index.jsindex 9f5950ec3..1d36341cb 100644--- a/src/recorder/index.js+++ b/src/recorder/index.js@@ -23,7 +23,9 @@ import { record } from 'rrweb';const REPLAY_FLUSH_EVENT_COUNT = 100;const REPLAY_FLUSH_INTERVAL = 2000;- const REPLAY_MAX_PAYLOAD_SIZE = 900000;+ const REPLAY_MAX_PAYLOAD_SIZE = 950000;+ const REPLAY_FRAGMENT_TYPE = 'umami:rrweb-event-fragment';+ const REPLAY_FRAGMENT_TOTAL_PLACEHOLDER = 999999999;const HEATMAP_FLUSH_EVENT_COUNT = 20;const HEATMAP_FLUSH_INTERVAL = 5000;@@ -41,6 +43,7 @@ import { record } from 'rrweb';let replayFlushTimer = null;let heatmapFlushTimer = null;let replayStartTime = null;+ let replayLastChunkIndex = 0;let replayStopped = false;let heatmapStarted = false;@@ -69,6 +72,35 @@ import { record } from 'rrweb';const isReplayPayloadTooLarge = (events, timestamp) =>getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE;+ const getReplayChunkIndex = () => {+ const timestamp = Math.floor(Date.now() / 1000);+ const chunkIndex = Math.max(timestamp, replayLastChunkIndex + 1);++ replayLastChunkIndex = chunkIndex;++ return chunkIndex;+ };++ const createReplayFragment = (id, index, total, timestamp, value) => ({+ type: REPLAY_FRAGMENT_TYPE,+ timestamp,+ data: {+ id,+ index,+ total,+ value,+ },+ });++ const getReplayEventTimestamp = event => {+ const timestamp = Number(event?.timestamp);++ return Number.isFinite(timestamp) ? timestamp : Date.now();+ };++ const getReplayFragmentId = () =>+ `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;+const sendPayload = (type, payload, useKeepalive = false) => {const cache = getSessionCache();@@ -90,45 +122,106 @@ import { record } from 'rrweb';}).catch(() => {});};+ const sendReplayChunk = (events, timestamp, useKeepalive = false) => {+ replayLastChunkIndex = Math.max(replayLastChunkIndex, timestamp);++ sendPayload(+ 'record',+ {+ events,+ timestamp,+ },+ useKeepalive,+ );+ };++ const getReplayEventFragments = (event, chunkTimestamp) => {+ const value = JSON.stringify(event);+ const id = getReplayFragmentId();+ const eventTimestamp = getReplayEventTimestamp(event);+ const fragments = [];+ let start = 0;++ while (start < value.length) {+ let low = start + 1;+ let high = value.length;+ let end = start;++ while (low <= high) {+ const mid = Math.floor((low + high) / 2);+ const fragmentChunkTimestamp = chunkTimestamp + fragments.length;+ const fragment = createReplayFragment(+ id,+ fragments.length,+ REPLAY_FRAGMENT_TOTAL_PLACEHOLDER,+ eventTimestamp,+ value.slice(start, mid),+ );++ if (isReplayPayloadTooLarge([fragment], fragmentChunkTimestamp)) {+ high = mid - 1;+ } else {+ end = mid;+ low = mid + 1;+ }+ }++ if (end === start) {+ end = start + 1;+ }++ fragments.push(value.slice(start, end));+ start = end;+ }++ return fragments.map((fragment, index) =>+ createReplayFragment(id, index, fragments.length, eventTimestamp, fragment),+ );+ };++ const sendReplayEventFragments = (event, timestamp, useKeepalive = false) => {+ const fragments = getReplayEventFragments(event, timestamp);++ fragments.forEach((fragment, index) => {+ sendReplayChunk([fragment], timestamp + index, useKeepalive);+ });++ return fragments.length;+ };+const sendReplayEvents = (events, timestamp, useKeepalive = false) => {let chunk = [];let chunkOffset = 0;- events.forEach(event => {- const candidate = [...chunk, event];+ for (const event of events) {+ const chunkTimestamp = timestamp + chunkOffset;- if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) {+ if (isReplayPayloadTooLarge([event], chunkTimestamp)) {if (chunk.length) {- sendPayload(- 'record',- {- events: chunk,- timestamp: timestamp + chunkOffset,- },- useKeepalive,- );+ sendReplayChunk(chunk, chunkTimestamp, useKeepalive);chunk = [];chunkOffset += 1;}- if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) {+ chunkOffset += sendReplayEventFragments(event, timestamp + chunkOffset, useKeepalive);+ continue;+ }++ const candidate = [...chunk, event];++ if (isReplayPayloadTooLarge(candidate, chunkTimestamp)) {+ if (chunk.length) {+ sendReplayChunk(chunk, chunkTimestamp, useKeepalive);+ chunk = [];chunkOffset += 1;- return;}}chunk.push(event);- });+ }if (chunk.length) {- sendPayload(- 'record',- {- events: chunk,- timestamp: timestamp + chunkOffset,- },- useKeepalive,- );+ sendReplayChunk(chunk, timestamp + chunkOffset, useKeepalive);}};@@ -138,7 +231,7 @@ import { record } from 'rrweb';const events = replayBuffer;replayBuffer = [];- sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive);+ sendReplayEvents(events, getReplayChunkIndex(), useKeepalive);};const flushHeatmap = (useKeepalive = false) => {@@ -326,10 +419,21 @@ import { record } from 'rrweb';return;}- if (- replayBuffer.length &&- isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000))- ) {+ const timestamp = Math.floor(Date.now() / 1000);++ if (isReplayPayloadTooLarge([event], timestamp)) {+ if (replayBuffer.length) {+ const events = replayBuffer;+ replayBuffer = [];++ sendReplayEvents(events, getReplayChunkIndex());+ }++ sendReplayEvents([event], getReplayChunkIndex());+ return;+ }++ if (replayBuffer.length && isReplayPayloadTooLarge([...replayBuffer, event], timestamp)) {flushReplay();}@@ -359,6 +463,11 @@ import { record } from 'rrweb';checkoutEveryNms: 30000,...(blockSelector && { blockSelector }),});++ if (replayStopped && replayStopFn) {+ replayStopFn();+ replayStopFn = null;+ }};const beginHeatmapCapture = () => {