🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
perf(replay): drop redundant serialization and parsing from replay ingest (#1316)
Goldflag829b2dc7194b
diff --git a/server/src/services/replay/replayClockSkew.test.ts b/server/src/services/replay/replayClockSkew.test.tsindex f12f8b9a1..9758d6d1e 100644--- a/server/src/services/replay/replayClockSkew.test.ts+++ b/server/src/services/replay/replayClockSkew.test.ts@@ -87,3 +87,85 @@ describe("correctReplayClockSkew", () => {expect(events[0].timestamp).toBe(NOW);});});++// The implementation before the single-pass fast path, kept verbatim so the+// optimisation is checked against it rather than against hand-written cases.+function referenceCorrection<T extends { timestamp: number }>(events: T[], nowMs: number) {+ if (events.length === 0) return { events, skewMs: 0 };+ const upperBound = nowMs + MAX_FUTURE_SKEW_MS;+ const lowerBound = nowMs - MAX_PAST_SKEW_MS;+ const sanitize = (t: number) =>+ Number.isFinite(t) ? t : t === Infinity ? upperBound : t === -Infinity ? lowerBound : nowMs;+ const timestamps = events.map(event => sanitize(event.timestamp));+ const sorted = [...timestamps].sort((a, b) => a - b);+ const mid = Math.floor(sorted.length / 2);+ const anchor = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];+ let skewMs = 0;+ if (anchor > upperBound) skewMs = anchor - nowMs;+ else if (anchor < lowerBound) skewMs = anchor - nowMs;+ const allFinite = events.every(event => Number.isFinite(event.timestamp));+ if (skewMs === 0 && allFinite && timestamps.every(t => t <= upperBound && t >= lowerBound)) {+ return { events, skewMs: 0 };+ }+ return {+ events: events.map((event, index) => ({+ ...event,+ timestamp: Math.min(Math.max(timestamps[index] - skewMs, lowerBound), upperBound),+ })),+ skewMs,+ };+}++describe("correctReplayClockSkew matches the reference implementation", () => {+ const upper = NOW + MAX_FUTURE_SKEW_MS;+ const lower = NOW - MAX_PAST_SKEW_MS;+ const pool = [+ NOW,+ NOW - 1,+ upper,+ lower,+ upper + 1,+ lower - 1,+ NOW + 10 * MAX_FUTURE_SKEW_MS,+ NOW - 10 * MAX_PAST_SKEW_MS,+ Date.UTC(2090, 6, 2),+ 0,+ Infinity,+ -Infinity,+ NaN,+ ];++ function check(timestamps: number[]) {+ const events = batch(...timestamps);+ const actual = correctReplayClockSkew(events, NOW);+ const expected = referenceCorrection(events, NOW);+ expect(actual).toEqual(expected);+ // Same identity contract: the input array comes back only when untouched.+ expect(actual.events === events).toBe(expected.events === events);+ }++ it("keeps the input array for a batch sitting exactly on both bounds", () => {+ const events = batch(lower, NOW, upper);+ expect(correctReplayClockSkew(events, NOW).events).toBe(events);+ });++ it("agrees on every one- and two-event combination of edge values", () => {+ for (const a of pool) {+ check([a]);+ for (const b of pool) check([a, b]);+ }+ });++ it("agrees on random mixed batches", () => {+ // Deterministic LCG so a failure reproduces.+ let seed = 12345;+ const next = () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31;+ for (let round = 0; round < 2000; round++) {+ const length = 1 + Math.floor(next() * 9);+ const timestamps = Array.from({ length }, () =>+ next() < 0.7 ? NOW - Math.floor(next() * MAX_PAST_SKEW_MS) : pool[Math.floor(next() * pool.length)]+ );+ check(timestamps);+ }+ });+});diff --git a/server/src/services/replay/replayClockSkew.ts b/server/src/services/replay/replayClockSkew.tsindex bdddd13f6..49324ad88 100644--- a/server/src/services/replay/replayClockSkew.ts+++ b/server/src/services/replay/replayClockSkew.ts@@ -61,6 +61,16 @@ export function correctReplayClockSkew<T extends { timestamp: number }>(const upperBound = nowMs + MAX_FUTURE_SKEW_MS;const lowerBound = nowMs - MAX_PAST_SKEW_MS;+ // Nearly every batch is healthy: each timestamp is finite and inside the+ // window, so the median is too and nothing would be corrected. One pass+ // settles that without copying or sorting the batch. NaN and ±Infinity fail+ // these comparisons, so they fall through to the full correction below.+ if (events.every(event => event.timestamp >= lowerBound && event.timestamp <= upperBound)) {+ return { events, skewMs: 0 };+ }++ // From here at least one timestamp is non-finite or out of range, so the+ // batch always comes back as a corrected copy.const timestamps = events.map(event => sanitize(event.timestamp, nowMs, lowerBound, upperBound));const anchor = median(timestamps);@@ -68,13 +78,6 @@ export function correctReplayClockSkew<T extends { timestamp: number }>(if (anchor > upperBound) skewMs = anchor - nowMs;else if (anchor < lowerBound) skewMs = anchor - nowMs;- // Returning the original array is only safe when nothing needed sanitizing;- // otherwise the caller would get the non-finite values straight back.- const allFinite = events.every(event => Number.isFinite(event.timestamp));- if (skewMs === 0 && allFinite && timestamps.every(t => t <= upperBound && t >= lowerBound)) {- return { events, skewMs: 0 };- }-return {events: events.map((event, index) => ({...event,diff --git a/server/src/services/replay/sessionReplayIngestService.test.ts b/server/src/services/replay/sessionReplayIngestService.test.tsindex e2f0ad67e..e93822365 100644--- a/server/src/services/replay/sessionReplayIngestService.test.ts+++ b/server/src/services/replay/sessionReplayIngestService.test.ts@@ -1,4 +1,5 @@import { beforeEach, describe, expect, it, vi } from "vitest";+import { toClickHouseDateTime } from "../../db/clickhouse/dateTime.js";import type { RecordSessionReplayRequest } from "../../types/sessionReplay.js";const mocks = vi.hoisted(() => ({@@ -163,3 +164,68 @@ describe("SessionReplayIngestService size cap", () => {expect(mocks.insert).not.toHaveBeenCalled();});});++describe("SessionReplayIngestService batch serialization", () => {+ // Inside the clock-skew window, so the batch is stored as sent.+ const base = Date.now() - 60_000;++ beforeEach(() => {+ vi.clearAllMocks();+ mocks.generateUserId.mockResolvedValue("fingerprint");+ mocks.updateSession.mockResolvedValue({ sessionId: "session-1" });+ mocks.insert.mockResolvedValue(undefined);+ mocks.chargeReplaySessionBudget.mockResolvedValue({ accepted: true, usedBytes: 0 });+ mocks.storeBatch.mockResolvedValue("r2-key");+ mocks.r2Enabled = true;+ });++ it("hands R2 the per-event strings, which join to the serialized data array", async () => {+ const service = new SessionReplayIngestService();+ const events = [+ { type: 2, data: { node: { id: 1, textContent: 'quote " and é' } }, timestamp: base },+ { type: 3, data: { source: 1, positions: [{ x: 1, y: 2 }] }, timestamp: base + 10 },+ { type: 4, data: null, timestamp: base + 20 },+ ];++ await service.recordEvents(42, { userId: "", events }, requestMeta);++ const serialized: string[] = mocks.storeBatch.mock.calls[0][2];+ expect(serialized).toEqual(events.map(event => JSON.stringify(event.data)));+ expect(`[${serialized.join(",")}]`).toBe(JSON.stringify(events.map(event => event.data)));+ expect(mocks.insert.mock.calls[0][0].values.map((row: { event_data_key: string }) => row.event_data_key)).toEqual([+ "r2-key",+ "r2-key",+ "r2-key",+ ]);+ });++ it("bounds the metadata row by the earliest and latest event, whatever their order", async () => {+ const service = new SessionReplayIngestService();+ const events = [+ { type: 3, data: {}, timestamp: base + 500 },+ { type: 3, data: {}, timestamp: base },+ { type: 3, data: {}, timestamp: base + 900 },+ { type: 3, data: {}, timestamp: base + 100 },+ ];++ await service.recordEvents(42, { userId: "", events, metadata: { pageUrl: "https://example.com/" } }, requestMeta);++ const metadataRow = mocks.insert.mock.calls[1][0].values[0];+ expect(metadataRow.start_time).toBe(toClickHouseDateTime(new Date(base)));+ expect(metadataRow.end_time).toBe(toClickHouseDateTime(new Date(base + 900)));+ expect(metadataRow.event_count).toBe(4);+ });++ it("records a batch too large to spread into Math.min", async () => {+ // Math.min(...timestamps) overflows the call stack somewhere past ~120k+ // arguments; a small-event batch that size still fits the body limit.+ const service = new SessionReplayIngestService();+ const events = Array.from({ length: 200_000 }, (_, index) => ({ type: 3, data: {}, timestamp: base + index }));++ await service.recordEvents(42, { userId: "", events, metadata: { pageUrl: "https://example.com/" } }, requestMeta);++ const metadataRow = mocks.insert.mock.calls[1][0].values[0];+ expect(metadataRow.start_time).toBe(toClickHouseDateTime(new Date(base)));+ expect(metadataRow.end_time).toBe(toClickHouseDateTime(new Date(base + 199_999)));+ });+});diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.tsindex d0b0e3b38..789545721 100644--- a/server/src/services/replay/sessionReplayIngestService.ts+++ b/server/src/services/replay/sessionReplayIngestService.ts@@ -34,6 +34,22 @@ export interface RequestMetadata {referrer: string;}+/**+ * Earliest and latest timestamp in a non-empty batch. A loop rather than+ * Math.min(...spread): spreading a large batch can exceed the engine's argument+ * limit and throw. Timestamps are finite here (correctReplayClockSkew sanitizes+ * them), so plain comparisons match Math.min/Math.max.+ */+function timestampRange(events: { timestamp: number }[]): { startMs: number; endMs: number } {+ let startMs = events[0].timestamp;+ let endMs = startMs;+ for (const { timestamp } of events) {+ if (timestamp < startMs) startMs = timestamp;+ if (timestamp > endMs) endMs = timestamp;+ }+ return { startMs, endMs };+}+/*** Service responsible for ingesting session replay data* Handles recording events and updating metadata@@ -90,15 +106,11 @@ export class SessionReplayIngestService {// Check if R2 storage is enabled for cloud deploymentslet r2BatchKey: string | null = null;- let eventDataArray: any[] = [];if (r2Storage.isEnabled()) {- // Extract event data for R2 storage- eventDataArray = events.map(event => event.data);-try {- // Store event data batch in R2- r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, eventDataArray);+ // Store event data batch in R2, reusing the strings serialized above+ r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, serializedEvents);} catch (error) {console.error("Failed to store in R2, falling back to ClickHouse:", error);r2BatchKey = null;@@ -162,10 +174,11 @@ export class SessionReplayIngestService {// An events-free batch still carries metadata worth recording, but it has// no timestamps to bound it — anchor those on server time rather than// letting Math.min of nothing produce an epoch-dated row.- const timestamps = events.length > 0 ? events.map(event => event.timestamp) : [Date.now()];+ const nowMs = Date.now();+ const { startMs, endMs } = events.length > 0 ? timestampRange(events) : { startMs: nowMs, endMs: nowMs };const batchStats: BatchStats = {- startTime: new Date(Math.min(...timestamps)),- endTime: new Date(Math.max(...timestamps)),+ startTime: new Date(startMs),+ endTime: new Date(endMs),eventCount: eventsToInsert.length,compressedSizeBytes: eventsToInsert.reduce((total, event) => total + event.event_size_bytes, 0),screenWidth: metadata.viewportWidth || 0,diff --git a/server/src/services/replay/trackingUtils.ts b/server/src/services/replay/trackingUtils.tsindex d9cd40bf7..62b3c6154 100644--- a/server/src/services/replay/trackingUtils.ts+++ b/server/src/services/replay/trackingUtils.ts@@ -1,6 +1,6 @@-import { UAParser } from "ua-parser-js";import { getLocation } from "../../db/geolocation/geolocation.js";import { getChannel } from "../tracker/getChannel.js";+import { parseUserAgent } from "../tracker/utils.js";import { getDeviceType } from "../../utils.js";export interface TrackingData {@@ -34,8 +34,9 @@ export async function parseTrackingData(screenWidth: number,screenHeight: number): Promise<TrackingData> {- // Parse user agent- const ua = UAParser(userAgent);+ // Parse user agent through the tracker's shared cache: replay batches from+ // one visitor repeat the same string, and the parse is regex-heavy+ const ua = parseUserAgent(userAgent);const geoData = await getLocation([ipAddress]);diff --git a/server/src/services/storage/r2StorageService.test.ts b/server/src/services/storage/r2StorageService.test.tsindex 756ba591e..7ea11435b 100644--- a/server/src/services/storage/r2StorageService.test.ts+++ b/server/src/services/storage/r2StorageService.test.ts@@ -69,6 +69,11 @@ async function loadStorage(options: { cloud?: boolean; accessKey?: string; secrereturn r2Storage;}+// What ingestion hands storeBatch: each event's data serialized on its own.+function serialize(values: unknown[]): string[] {+ return values.map(value => JSON.stringify(value));+}+function sentInput(call = 0): Record<string, any> {return mocks.send.mock.calls[call][0].input;}@@ -97,7 +102,7 @@ describe("r2Storage initialization", () => {const storage = await loadStorage({ cloud: false });expect(storage.isEnabled()).toBe(false);- await expect(storage.storeBatch(1, "session", [{ type: "pageview" }])).resolves.toBeNull();+ await expect(storage.storeBatch(1, "session", [JSON.stringify({ type: "pageview" })])).resolves.toBeNull();await expect(storage.getBatch("1/session/batch.json.zst")).rejects.toThrow("R2 storage is not enabled");await expect(storage.deleteBatch("1/session/batch.json.zst")).resolves.toBeUndefined();expect(mocks.clientConfigs).toHaveLength(0);@@ -164,7 +169,7 @@ describe("r2Storage.storeBatch", () => {{ type: "custom_event", name: "signup" },];- await expect(storage.storeBatch(42, "session/with spaces", events)).resolves.toBe(+ await expect(storage.storeBatch(42, "session/with spaces", serialize(events))).resolves.toBe("42/session/with spaces/1725000123456.json.zst");@@ -185,6 +190,31 @@ describe("r2Storage.storeBatch", () => {expect(sentInput().Body).toEqual(Buffer.from(`zstd:${JSON.stringify(events)}`));});+ it("writes the same bytes as serializing the whole data array", async () => {+ mocks.send.mockResolvedValue({});+ const storage = await loadStorage();+ // Shapes rrweb and the JSON body parser can produce, including the+ // characters JSON.stringify escapes and non-ASCII text.+ const data: unknown[] = [+ { type: 2, data: { node: { childNodes: [{ textContent: 'quote " backslash \\ newline \n tab \t' }] } } },+ { source: 3, positions: [{ x: 1.5, y: -2, timeOffset: 0 }] },+ null,+ 0,+ -12.75,+ true,+ "plain string",+ "é ünïcödé 日本語 🐸 \u2028\u2029",+ [],+ {},+ [[1, [2, [3]]], { nested: { deeper: [null, false, ""] } }],+ { "key with spaces": 1, "": "empty key", "__proto__-ish": "x" },+ ];++ await storage.storeBatch(7, "session", serialize(data));++ expect(mocks.compress.mock.calls[0][0]).toEqual(Buffer.from(JSON.stringify(data)));+ });+it("supports empty batches and records a zero event count", async () => {vi.spyOn(Date, "now").mockReturnValue(100);mocks.send.mockResolvedValue({});@@ -192,6 +222,7 @@ describe("r2Storage.storeBatch", () => {await expect(storage.storeBatch(0, "empty", [])).resolves.toBe("0/empty/100.json.zst");expect(sentInput().Metadata.eventCount).toBe("0");+ expect(mocks.compress.mock.calls[0][0].toString()).toBe(JSON.stringify([]));});it("propagates compression failures without issuing a put", async () => {@@ -199,7 +230,7 @@ describe("r2Storage.storeBatch", () => {mocks.compress.mockRejectedValue(compressionError);const storage = await loadStorage();- await expect(storage.storeBatch(1, "session", [{}])).rejects.toBe(compressionError);+ await expect(storage.storeBatch(1, "session", ["{}"])).rejects.toBe(compressionError);expect(mocks.send).not.toHaveBeenCalled();expect(console.error).toHaveBeenCalledWith("[R2Storage] Failed to store batch:", compressionError);});@@ -209,7 +240,7 @@ describe("r2Storage.storeBatch", () => {mocks.send.mockRejectedValue(putError);const storage = await loadStorage();- await expect(storage.storeBatch(1, "session", [{}])).rejects.toBe(putError);+ await expect(storage.storeBatch(1, "session", ["{}"])).rejects.toBe(putError);expect(mocks.compress).toHaveBeenCalledOnce();expect(console.error).toHaveBeenCalledWith("[R2Storage] Failed to store batch:", putError);});@@ -224,8 +255,8 @@ describe("r2Storage.storeBatch", () => {mocks.send.mockResolvedValue({});const storage = await loadStorage();- const first = storage.storeBatch(1, "first", [{ sequence: 1 }]);- const second = storage.storeBatch(2, "second", [{ sequence: 2 }]);+ const first = storage.storeBatch(1, "first", [JSON.stringify({ sequence: 1 })]);+ const second = storage.storeBatch(2, "second", [JSON.stringify({ sequence: 2 })]);await expect(second).resolves.toBe("2/second/500.json.zst");expect(mocks.send).toHaveBeenCalledOnce();diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.tsindex 921a0dc9e..198480524 100644--- a/server/src/services/storage/r2StorageService.ts+++ b/server/src/services/storage/r2StorageService.ts@@ -67,8 +67,13 @@ class R2StorageService {/*** Store a batch of event data in R2* Returns the storage key if successful, null if R2 is disabled+ *+ * Takes each event's data already serialized (`JSON.stringify(event.data)`),+ * which ingestion computes anyway to size the batch. Joining those strings+ * into an array literal yields the same bytes as `JSON.stringify` of the data+ * array, without serializing the whole batch a second time.*/- async storeBatch(siteId: number, sessionId: string, eventDataArray: any[]): Promise<string | null> {+ async storeBatch(siteId: number, sessionId: string, serializedEvents: string[]): Promise<string | null> {if (!this.enabled || !this.client) {return null;}@@ -78,7 +83,7 @@ class R2StorageService {try {// Compress with zstd - much faster decompression than brotli- const jsonBuffer = Buffer.from(JSON.stringify(eventDataArray));+ const jsonBuffer = Buffer.from(`[${serializedEvents.join(",")}]`);const compressed = await zstdCompress(jsonBuffer, 3); // level 3 = good balanceawait this.client.send(@@ -91,7 +96,7 @@ class R2StorageService {Metadata: {siteId: siteId.toString(),sessionId: sessionId,- eventCount: eventDataArray.length.toString(),+ eventCount: serializedEvents.length.toString(),compression: "zstd",},})