Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Reject oversized replay payloads and chunk recorder flushes
Mike Cao475d9548de59
diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.tsindex ad304a27c..7fa267b79 100644--- a/src/app/api/record/route.ts+++ b/src/app/api/record/route.ts@@ -8,7 +8,7 @@ import { parseToken } from '@/lib/jwt';import { fetchAccount, fetchTeam } from '@/lib/load';import { getRecorderConfig } from '@/lib/recorder';import { parseRequest } from '@/lib/request';-import { badRequest, forbidden, json, serverError } from '@/lib/response';+import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response';import { getWebsite } from '@/queries/prisma';import { saveRecording } from '@/queries/sql';import { saveHeatmapEvents } from '@/queries/sql/heatmap/saveHeatmapEvents';@@ -18,6 +18,8 @@ interface Cache {visitId: string;}+const MAX_RECORD_REQUEST_BYTES = 1_000_000;+const schema = z.discriminatedUnion('type', [z.object({type: z.literal('record'),@@ -73,8 +75,38 @@ function getUrlPath(url: string) {}}+async function getRequestBodySize(request: Request): Promise<number | null> {+ const contentLength = request.headers.get('content-length');++ if (contentLength) {+ const size = Number(contentLength);++ if (Number.isFinite(size)) {+ return size;+ }+ }++ try {+ const text = await request.clone().text();++ return new TextEncoder().encode(text).length;+ } catch {+ return null;+ }+}+export async function POST(request: Request) {try {+ const requestBodySize = await getRequestBodySize(request);++ if (requestBodySize && requestBodySize > MAX_RECORD_REQUEST_BYTES) {+ return payloadTooLarge({+ reason: 'payload_too_large',+ maxBytes: MAX_RECORD_REQUEST_BYTES,+ size: requestBodySize,+ });+ }+const { body, error } = await parseRequest(request, schema, { skipAuth: true });if (error) {diff --git a/src/lib/response.ts b/src/lib/response.tsindex f1ad5c7bf..587802619 100644--- a/src/lib/response.ts+++ b/src/lib/response.ts@@ -36,6 +36,15 @@ export function forbidden(error?: Record<string, any>) {);}+export function payloadTooLarge(error?: Record<string, any>) {+ return Response.json(+ {+ error: { message: 'Payload too large', code: 'payload-too-large', status: 413, ...error },+ },+ { status: 413 },+ );+}+export function notFound(error?: Record<string, any>) {return Response.json({ error: { message: 'Not found', code: 'not-found', status: 404, ...error } },diff --git a/src/recorder/index.js b/src/recorder/index.jsindex e22ab9b98..9f5950ec3 100644--- a/src/recorder/index.js+++ b/src/recorder/index.js@@ -23,6 +23,7 @@ import { record } from 'rrweb';const REPLAY_FLUSH_EVENT_COUNT = 100;const REPLAY_FLUSH_INTERVAL = 2000;+ const REPLAY_MAX_PAYLOAD_SIZE = 900000;const HEATMAP_FLUSH_EVENT_COUNT = 20;const HEATMAP_FLUSH_INTERVAL = 5000;@@ -45,12 +46,8 @@ import { record } from 'rrweb';const getSessionCache = () => window.umami?.getSession?.()?.cache;- const sendPayload = (type, payload, useKeepalive = false) => {- const cache = getSessionCache();-- if (!cache) return;-- const body = JSON.stringify({+ const getPayloadBody = (type, payload) =>+ JSON.stringify({type,payload: {website,@@ -58,7 +55,28 @@ import { record } from 'rrweb';},});- const keepalive = useKeepalive && body.length < 60000;+ const getPayloadSize = body => {+ try {+ return new Blob([body]).size;+ } catch {+ return body.length;+ }+ };++ const getReplayPayloadSize = (events, timestamp) =>+ getPayloadSize(getPayloadBody('record', { events, timestamp }));++ const isReplayPayloadTooLarge = (events, timestamp) =>+ getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE;++ const sendPayload = (type, payload, useKeepalive = false) => {+ const cache = getSessionCache();++ if (!cache) return;++ const body = getPayloadBody(type, payload);++ const keepalive = useKeepalive && getPayloadSize(body) < 60000;return fetch(endpoint, {keepalive,@@ -72,20 +90,55 @@ import { record } from 'rrweb';}).catch(() => {});};+ const sendReplayEvents = (events, timestamp, useKeepalive = false) => {+ let chunk = [];+ let chunkOffset = 0;++ events.forEach(event => {+ const candidate = [...chunk, event];++ if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) {+ if (chunk.length) {+ sendPayload(+ 'record',+ {+ events: chunk,+ timestamp: timestamp + chunkOffset,+ },+ useKeepalive,+ );+ chunk = [];+ chunkOffset += 1;+ }++ if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) {+ chunkOffset += 1;+ return;+ }+ }++ chunk.push(event);+ });++ if (chunk.length) {+ sendPayload(+ 'record',+ {+ events: chunk,+ timestamp: timestamp + chunkOffset,+ },+ useKeepalive,+ );+ }+ };+const flushReplay = (useKeepalive = false) => {if (!replayBuffer.length) return;const events = replayBuffer;replayBuffer = [];- sendPayload(- 'record',- {- events,- timestamp: Math.floor(Date.now() / 1000),- },- useKeepalive,- );+ sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive);};const flushHeatmap = (useKeepalive = false) => {@@ -273,9 +326,19 @@ import { record } from 'rrweb';return;}+ if (+ replayBuffer.length &&+ isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000))+ ) {+ flushReplay();+ }+replayBuffer.push(event);- if (replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT) {+ if (+ replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT ||+ isReplayPayloadTooLarge(replayBuffer, Math.floor(Date.now() / 1000))+ ) {flushReplay();}},