Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Reject oversized replay payloads and chunk recorder flushes

Mike Cao475d9548de59
diff --git a/src/app/api/record/route.ts b/src/app/api/record/route.ts
index ad304a27c..7fa267b79 100644
--- a/src/app/api/record/route.ts
+++ b/src/app/api/record/route.ts
@@ -8,7 +8,7 @@ import { parseToken } from '@/lib/jwt';
import { fetchAccount, fetchTeam } from '@/lib/load';
import { getRecorderConfig } from '@/lib/recorder';
import { parseRequest } from '@/lib/request';
-import { badRequest, forbidden, json, serverError } from '@/lib/response';
+import { badRequest, forbidden, json, payloadTooLarge, serverError } from '@/lib/response';
import { getWebsite } from '@/queries/prisma';
import { saveRecording } from '@/queries/sql';
import { saveHeatmapEvents } from '@/queries/sql/heatmap/saveHeatmapEvents';
@@ -18,6 +18,8 @@ interface Cache {
visitId: string;
}
+const MAX_RECORD_REQUEST_BYTES = 1_000_000;
+
const schema = z.discriminatedUnion('type', [
z.object({
type: z.literal('record'),
@@ -73,8 +75,38 @@ function getUrlPath(url: string) {
}
}
+async function getRequestBodySize(request: Request): Promise<number | null> {
+ const contentLength = request.headers.get('content-length');
+
+ if (contentLength) {
+ const size = Number(contentLength);
+
+ if (Number.isFinite(size)) {
+ return size;
+ }
+ }
+
+ try {
+ const text = await request.clone().text();
+
+ return new TextEncoder().encode(text).length;
+ } catch {
+ return null;
+ }
+}
+
export async function POST(request: Request) {
try {
+ const requestBodySize = await getRequestBodySize(request);
+
+ if (requestBodySize && requestBodySize > MAX_RECORD_REQUEST_BYTES) {
+ return payloadTooLarge({
+ reason: 'payload_too_large',
+ maxBytes: MAX_RECORD_REQUEST_BYTES,
+ size: requestBodySize,
+ });
+ }
+
const { body, error } = await parseRequest(request, schema, { skipAuth: true });
if (error) {
diff --git a/src/lib/response.ts b/src/lib/response.ts
index f1ad5c7bf..587802619 100644
--- a/src/lib/response.ts
+++ b/src/lib/response.ts
@@ -36,6 +36,15 @@ export function forbidden(error?: Record<string, any>) {
);
}
+export function payloadTooLarge(error?: Record<string, any>) {
+ return Response.json(
+ {
+ error: { message: 'Payload too large', code: 'payload-too-large', status: 413, ...error },
+ },
+ { status: 413 },
+ );
+}
+
export function notFound(error?: Record<string, any>) {
return Response.json(
{ error: { message: 'Not found', code: 'not-found', status: 404, ...error } },
diff --git a/src/recorder/index.js b/src/recorder/index.js
index e22ab9b98..9f5950ec3 100644
--- a/src/recorder/index.js
+++ b/src/recorder/index.js
@@ -23,6 +23,7 @@ import { record } from 'rrweb';
const REPLAY_FLUSH_EVENT_COUNT = 100;
const REPLAY_FLUSH_INTERVAL = 2000;
+ const REPLAY_MAX_PAYLOAD_SIZE = 900000;
const HEATMAP_FLUSH_EVENT_COUNT = 20;
const HEATMAP_FLUSH_INTERVAL = 5000;
@@ -45,12 +46,8 @@ import { record } from 'rrweb';
const getSessionCache = () => window.umami?.getSession?.()?.cache;
- const sendPayload = (type, payload, useKeepalive = false) => {
- const cache = getSessionCache();
-
- if (!cache) return;
-
- const body = JSON.stringify({
+ const getPayloadBody = (type, payload) =>
+ JSON.stringify({
type,
payload: {
website,
@@ -58,7 +55,28 @@ import { record } from 'rrweb';
},
});
- const keepalive = useKeepalive && body.length < 60000;
+ const getPayloadSize = body => {
+ try {
+ return new Blob([body]).size;
+ } catch {
+ return body.length;
+ }
+ };
+
+ const getReplayPayloadSize = (events, timestamp) =>
+ getPayloadSize(getPayloadBody('record', { events, timestamp }));
+
+ const isReplayPayloadTooLarge = (events, timestamp) =>
+ getReplayPayloadSize(events, timestamp) > REPLAY_MAX_PAYLOAD_SIZE;
+
+ const sendPayload = (type, payload, useKeepalive = false) => {
+ const cache = getSessionCache();
+
+ if (!cache) return;
+
+ const body = getPayloadBody(type, payload);
+
+ const keepalive = useKeepalive && getPayloadSize(body) < 60000;
return fetch(endpoint, {
keepalive,
@@ -72,20 +90,55 @@ import { record } from 'rrweb';
}).catch(() => {});
};
+ const sendReplayEvents = (events, timestamp, useKeepalive = false) => {
+ let chunk = [];
+ let chunkOffset = 0;
+
+ events.forEach(event => {
+ const candidate = [...chunk, event];
+
+ if (isReplayPayloadTooLarge(candidate, timestamp + chunkOffset)) {
+ if (chunk.length) {
+ sendPayload(
+ 'record',
+ {
+ events: chunk,
+ timestamp: timestamp + chunkOffset,
+ },
+ useKeepalive,
+ );
+ chunk = [];
+ chunkOffset += 1;
+ }
+
+ if (isReplayPayloadTooLarge([event], timestamp + chunkOffset)) {
+ chunkOffset += 1;
+ return;
+ }
+ }
+
+ chunk.push(event);
+ });
+
+ if (chunk.length) {
+ sendPayload(
+ 'record',
+ {
+ events: chunk,
+ timestamp: timestamp + chunkOffset,
+ },
+ useKeepalive,
+ );
+ }
+ };
+
const flushReplay = (useKeepalive = false) => {
if (!replayBuffer.length) return;
const events = replayBuffer;
replayBuffer = [];
- sendPayload(
- 'record',
- {
- events,
- timestamp: Math.floor(Date.now() / 1000),
- },
- useKeepalive,
- );
+ sendReplayEvents(events, Math.floor(Date.now() / 1000), useKeepalive);
};
const flushHeatmap = (useKeepalive = false) => {
@@ -273,9 +326,19 @@ import { record } from 'rrweb';
return;
}
+ if (
+ replayBuffer.length &&
+ isReplayPayloadTooLarge([...replayBuffer, event], Math.floor(Date.now() / 1000))
+ ) {
+ flushReplay();
+ }
+
replayBuffer.push(event);
- if (replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT) {
+ if (
+ replayBuffer.length >= REPLAY_FLUSH_EVENT_COUNT ||
+ isReplayPayloadTooLarge(replayBuffer, Math.floor(Date.now() / 1000))
+ ) {
flushReplay();
}
},