Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

perf(replay): drop redundant serialization and parsing from replay ingest (#1316)

Goldflag829b2dc7194b
diff --git a/server/src/services/replay/replayClockSkew.test.ts b/server/src/services/replay/replayClockSkew.test.ts
index f12f8b9a1..9758d6d1e 100644
--- a/server/src/services/replay/replayClockSkew.test.ts
+++ b/server/src/services/replay/replayClockSkew.test.ts
@@ -87,3 +87,85 @@ describe("correctReplayClockSkew", () => {
expect(events[0].timestamp).toBe(NOW);
});
});
+
+// The implementation before the single-pass fast path, kept verbatim so the
+// optimisation is checked against it rather than against hand-written cases.
+function referenceCorrection<T extends { timestamp: number }>(events: T[], nowMs: number) {
+ if (events.length === 0) return { events, skewMs: 0 };
+ const upperBound = nowMs + MAX_FUTURE_SKEW_MS;
+ const lowerBound = nowMs - MAX_PAST_SKEW_MS;
+ const sanitize = (t: number) =>
+ Number.isFinite(t) ? t : t === Infinity ? upperBound : t === -Infinity ? lowerBound : nowMs;
+ const timestamps = events.map(event => sanitize(event.timestamp));
+ const sorted = [...timestamps].sort((a, b) => a - b);
+ const mid = Math.floor(sorted.length / 2);
+ const anchor = sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+ let skewMs = 0;
+ if (anchor > upperBound) skewMs = anchor - nowMs;
+ else if (anchor < lowerBound) skewMs = anchor - nowMs;
+ const allFinite = events.every(event => Number.isFinite(event.timestamp));
+ if (skewMs === 0 && allFinite && timestamps.every(t => t <= upperBound && t >= lowerBound)) {
+ return { events, skewMs: 0 };
+ }
+ return {
+ events: events.map((event, index) => ({
+ ...event,
+ timestamp: Math.min(Math.max(timestamps[index] - skewMs, lowerBound), upperBound),
+ })),
+ skewMs,
+ };
+}
+
+describe("correctReplayClockSkew matches the reference implementation", () => {
+ const upper = NOW + MAX_FUTURE_SKEW_MS;
+ const lower = NOW - MAX_PAST_SKEW_MS;
+ const pool = [
+ NOW,
+ NOW - 1,
+ upper,
+ lower,
+ upper + 1,
+ lower - 1,
+ NOW + 10 * MAX_FUTURE_SKEW_MS,
+ NOW - 10 * MAX_PAST_SKEW_MS,
+ Date.UTC(2090, 6, 2),
+ 0,
+ Infinity,
+ -Infinity,
+ NaN,
+ ];
+
+ function check(timestamps: number[]) {
+ const events = batch(...timestamps);
+ const actual = correctReplayClockSkew(events, NOW);
+ const expected = referenceCorrection(events, NOW);
+ expect(actual).toEqual(expected);
+ // Same identity contract: the input array comes back only when untouched.
+ expect(actual.events === events).toBe(expected.events === events);
+ }
+
+ it("keeps the input array for a batch sitting exactly on both bounds", () => {
+ const events = batch(lower, NOW, upper);
+ expect(correctReplayClockSkew(events, NOW).events).toBe(events);
+ });
+
+ it("agrees on every one- and two-event combination of edge values", () => {
+ for (const a of pool) {
+ check([a]);
+ for (const b of pool) check([a, b]);
+ }
+ });
+
+ it("agrees on random mixed batches", () => {
+ // Deterministic LCG so a failure reproduces.
+ let seed = 12345;
+ const next = () => (seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31;
+ for (let round = 0; round < 2000; round++) {
+ const length = 1 + Math.floor(next() * 9);
+ const timestamps = Array.from({ length }, () =>
+ next() < 0.7 ? NOW - Math.floor(next() * MAX_PAST_SKEW_MS) : pool[Math.floor(next() * pool.length)]
+ );
+ check(timestamps);
+ }
+ });
+});
diff --git a/server/src/services/replay/replayClockSkew.ts b/server/src/services/replay/replayClockSkew.ts
index bdddd13f6..49324ad88 100644
--- a/server/src/services/replay/replayClockSkew.ts
+++ b/server/src/services/replay/replayClockSkew.ts
@@ -61,6 +61,16 @@ export function correctReplayClockSkew<T extends { timestamp: number }>(
const upperBound = nowMs + MAX_FUTURE_SKEW_MS;
const lowerBound = nowMs - MAX_PAST_SKEW_MS;
+ // Nearly every batch is healthy: each timestamp is finite and inside the
+ // window, so the median is too and nothing would be corrected. One pass
+ // settles that without copying or sorting the batch. NaN and ±Infinity fail
+ // these comparisons, so they fall through to the full correction below.
+ if (events.every(event => event.timestamp >= lowerBound && event.timestamp <= upperBound)) {
+ return { events, skewMs: 0 };
+ }
+
+ // From here at least one timestamp is non-finite or out of range, so the
+ // batch always comes back as a corrected copy.
const timestamps = events.map(event => sanitize(event.timestamp, nowMs, lowerBound, upperBound));
const anchor = median(timestamps);
@@ -68,13 +78,6 @@ export function correctReplayClockSkew<T extends { timestamp: number }>(
if (anchor > upperBound) skewMs = anchor - nowMs;
else if (anchor < lowerBound) skewMs = anchor - nowMs;
- // Returning the original array is only safe when nothing needed sanitizing;
- // otherwise the caller would get the non-finite values straight back.
- const allFinite = events.every(event => Number.isFinite(event.timestamp));
- if (skewMs === 0 && allFinite && timestamps.every(t => t <= upperBound && t >= lowerBound)) {
- return { events, skewMs: 0 };
- }
-
return {
events: events.map((event, index) => ({
...event,
diff --git a/server/src/services/replay/sessionReplayIngestService.test.ts b/server/src/services/replay/sessionReplayIngestService.test.ts
index e2f0ad67e..e93822365 100644
--- a/server/src/services/replay/sessionReplayIngestService.test.ts
+++ b/server/src/services/replay/sessionReplayIngestService.test.ts
@@ -1,4 +1,5 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import { toClickHouseDateTime } from "../../db/clickhouse/dateTime.js";
import type { RecordSessionReplayRequest } from "../../types/sessionReplay.js";
const mocks = vi.hoisted(() => ({
@@ -163,3 +164,68 @@ describe("SessionReplayIngestService size cap", () => {
expect(mocks.insert).not.toHaveBeenCalled();
});
});
+
+describe("SessionReplayIngestService batch serialization", () => {
+ // Inside the clock-skew window, so the batch is stored as sent.
+ const base = Date.now() - 60_000;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.generateUserId.mockResolvedValue("fingerprint");
+ mocks.updateSession.mockResolvedValue({ sessionId: "session-1" });
+ mocks.insert.mockResolvedValue(undefined);
+ mocks.chargeReplaySessionBudget.mockResolvedValue({ accepted: true, usedBytes: 0 });
+ mocks.storeBatch.mockResolvedValue("r2-key");
+ mocks.r2Enabled = true;
+ });
+
+ it("hands R2 the per-event strings, which join to the serialized data array", async () => {
+ const service = new SessionReplayIngestService();
+ const events = [
+ { type: 2, data: { node: { id: 1, textContent: 'quote " and é' } }, timestamp: base },
+ { type: 3, data: { source: 1, positions: [{ x: 1, y: 2 }] }, timestamp: base + 10 },
+ { type: 4, data: null, timestamp: base + 20 },
+ ];
+
+ await service.recordEvents(42, { userId: "", events }, requestMeta);
+
+ const serialized: string[] = mocks.storeBatch.mock.calls[0][2];
+ expect(serialized).toEqual(events.map(event => JSON.stringify(event.data)));
+ expect(`[${serialized.join(",")}]`).toBe(JSON.stringify(events.map(event => event.data)));
+ expect(mocks.insert.mock.calls[0][0].values.map((row: { event_data_key: string }) => row.event_data_key)).toEqual([
+ "r2-key",
+ "r2-key",
+ "r2-key",
+ ]);
+ });
+
+ it("bounds the metadata row by the earliest and latest event, whatever their order", async () => {
+ const service = new SessionReplayIngestService();
+ const events = [
+ { type: 3, data: {}, timestamp: base + 500 },
+ { type: 3, data: {}, timestamp: base },
+ { type: 3, data: {}, timestamp: base + 900 },
+ { type: 3, data: {}, timestamp: base + 100 },
+ ];
+
+ await service.recordEvents(42, { userId: "", events, metadata: { pageUrl: "https://example.com/" } }, requestMeta);
+
+ const metadataRow = mocks.insert.mock.calls[1][0].values[0];
+ expect(metadataRow.start_time).toBe(toClickHouseDateTime(new Date(base)));
+ expect(metadataRow.end_time).toBe(toClickHouseDateTime(new Date(base + 900)));
+ expect(metadataRow.event_count).toBe(4);
+ });
+
+ it("records a batch too large to spread into Math.min", async () => {
+ // Math.min(...timestamps) overflows the call stack somewhere past ~120k
+ // arguments; a small-event batch that size still fits the body limit.
+ const service = new SessionReplayIngestService();
+ const events = Array.from({ length: 200_000 }, (_, index) => ({ type: 3, data: {}, timestamp: base + index }));
+
+ await service.recordEvents(42, { userId: "", events, metadata: { pageUrl: "https://example.com/" } }, requestMeta);
+
+ const metadataRow = mocks.insert.mock.calls[1][0].values[0];
+ expect(metadataRow.start_time).toBe(toClickHouseDateTime(new Date(base)));
+ expect(metadataRow.end_time).toBe(toClickHouseDateTime(new Date(base + 199_999)));
+ });
+});
diff --git a/server/src/services/replay/sessionReplayIngestService.ts b/server/src/services/replay/sessionReplayIngestService.ts
index d0b0e3b38..789545721 100644
--- a/server/src/services/replay/sessionReplayIngestService.ts
+++ b/server/src/services/replay/sessionReplayIngestService.ts
@@ -34,6 +34,22 @@ export interface RequestMetadata {
referrer: string;
}
+/**
+ * Earliest and latest timestamp in a non-empty batch. A loop rather than
+ * Math.min(...spread): spreading a large batch can exceed the engine's argument
+ * limit and throw. Timestamps are finite here (correctReplayClockSkew sanitizes
+ * them), so plain comparisons match Math.min/Math.max.
+ */
+function timestampRange(events: { timestamp: number }[]): { startMs: number; endMs: number } {
+ let startMs = events[0].timestamp;
+ let endMs = startMs;
+ for (const { timestamp } of events) {
+ if (timestamp < startMs) startMs = timestamp;
+ if (timestamp > endMs) endMs = timestamp;
+ }
+ return { startMs, endMs };
+}
+
/**
* Service responsible for ingesting session replay data
* Handles recording events and updating metadata
@@ -90,15 +106,11 @@ export class SessionReplayIngestService {
// Check if R2 storage is enabled for cloud deployments
let r2BatchKey: string | null = null;
- let eventDataArray: any[] = [];
if (r2Storage.isEnabled()) {
- // Extract event data for R2 storage
- eventDataArray = events.map(event => event.data);
-
try {
- // Store event data batch in R2
- r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, eventDataArray);
+ // Store event data batch in R2, reusing the strings serialized above
+ r2BatchKey = await r2Storage.storeBatch(siteId, sessionId, serializedEvents);
} catch (error) {
console.error("Failed to store in R2, falling back to ClickHouse:", error);
r2BatchKey = null;
@@ -162,10 +174,11 @@ export class SessionReplayIngestService {
// An events-free batch still carries metadata worth recording, but it has
// no timestamps to bound it — anchor those on server time rather than
// letting Math.min of nothing produce an epoch-dated row.
- const timestamps = events.length > 0 ? events.map(event => event.timestamp) : [Date.now()];
+ const nowMs = Date.now();
+ const { startMs, endMs } = events.length > 0 ? timestampRange(events) : { startMs: nowMs, endMs: nowMs };
const batchStats: BatchStats = {
- startTime: new Date(Math.min(...timestamps)),
- endTime: new Date(Math.max(...timestamps)),
+ startTime: new Date(startMs),
+ endTime: new Date(endMs),
eventCount: eventsToInsert.length,
compressedSizeBytes: eventsToInsert.reduce((total, event) => total + event.event_size_bytes, 0),
screenWidth: metadata.viewportWidth || 0,
diff --git a/server/src/services/replay/trackingUtils.ts b/server/src/services/replay/trackingUtils.ts
index d9cd40bf7..62b3c6154 100644
--- a/server/src/services/replay/trackingUtils.ts
+++ b/server/src/services/replay/trackingUtils.ts
@@ -1,6 +1,6 @@
-import { UAParser } from "ua-parser-js";
import { getLocation } from "../../db/geolocation/geolocation.js";
import { getChannel } from "../tracker/getChannel.js";
+import { parseUserAgent } from "../tracker/utils.js";
import { getDeviceType } from "../../utils.js";
export interface TrackingData {
@@ -34,8 +34,9 @@ export async function parseTrackingData(
screenWidth: number,
screenHeight: number
): Promise<TrackingData> {
- // Parse user agent
- const ua = UAParser(userAgent);
+ // Parse user agent through the tracker's shared cache: replay batches from
+ // one visitor repeat the same string, and the parse is regex-heavy
+ const ua = parseUserAgent(userAgent);
const geoData = await getLocation([ipAddress]);
diff --git a/server/src/services/storage/r2StorageService.test.ts b/server/src/services/storage/r2StorageService.test.ts
index 756ba591e..7ea11435b 100644
--- a/server/src/services/storage/r2StorageService.test.ts
+++ b/server/src/services/storage/r2StorageService.test.ts
@@ -69,6 +69,11 @@ async function loadStorage(options: { cloud?: boolean; accessKey?: string; secre
return r2Storage;
}
+// What ingestion hands storeBatch: each event's data serialized on its own.
+function serialize(values: unknown[]): string[] {
+ return values.map(value => JSON.stringify(value));
+}
+
function sentInput(call = 0): Record<string, any> {
return mocks.send.mock.calls[call][0].input;
}
@@ -97,7 +102,7 @@ describe("r2Storage initialization", () => {
const storage = await loadStorage({ cloud: false });
expect(storage.isEnabled()).toBe(false);
- await expect(storage.storeBatch(1, "session", [{ type: "pageview" }])).resolves.toBeNull();
+ await expect(storage.storeBatch(1, "session", [JSON.stringify({ type: "pageview" })])).resolves.toBeNull();
await expect(storage.getBatch("1/session/batch.json.zst")).rejects.toThrow("R2 storage is not enabled");
await expect(storage.deleteBatch("1/session/batch.json.zst")).resolves.toBeUndefined();
expect(mocks.clientConfigs).toHaveLength(0);
@@ -164,7 +169,7 @@ describe("r2Storage.storeBatch", () => {
{ type: "custom_event", name: "signup" },
];
- await expect(storage.storeBatch(42, "session/with spaces", events)).resolves.toBe(
+ await expect(storage.storeBatch(42, "session/with spaces", serialize(events))).resolves.toBe(
"42/session/with spaces/1725000123456.json.zst"
);
@@ -185,6 +190,31 @@ describe("r2Storage.storeBatch", () => {
expect(sentInput().Body).toEqual(Buffer.from(`zstd:${JSON.stringify(events)}`));
});
+ it("writes the same bytes as serializing the whole data array", async () => {
+ mocks.send.mockResolvedValue({});
+ const storage = await loadStorage();
+ // Shapes rrweb and the JSON body parser can produce, including the
+ // characters JSON.stringify escapes and non-ASCII text.
+ const data: unknown[] = [
+ { type: 2, data: { node: { childNodes: [{ textContent: 'quote " backslash \\ newline \n tab \t' }] } } },
+ { source: 3, positions: [{ x: 1.5, y: -2, timeOffset: 0 }] },
+ null,
+ 0,
+ -12.75,
+ true,
+ "plain string",
+ "é ünïcödé 日本語 🐸 \u2028\u2029",
+ [],
+ {},
+ [[1, [2, [3]]], { nested: { deeper: [null, false, ""] } }],
+ { "key with spaces": 1, "": "empty key", "__proto__-ish": "x" },
+ ];
+
+ await storage.storeBatch(7, "session", serialize(data));
+
+ expect(mocks.compress.mock.calls[0][0]).toEqual(Buffer.from(JSON.stringify(data)));
+ });
+
it("supports empty batches and records a zero event count", async () => {
vi.spyOn(Date, "now").mockReturnValue(100);
mocks.send.mockResolvedValue({});
@@ -192,6 +222,7 @@ describe("r2Storage.storeBatch", () => {
await expect(storage.storeBatch(0, "empty", [])).resolves.toBe("0/empty/100.json.zst");
expect(sentInput().Metadata.eventCount).toBe("0");
+ expect(mocks.compress.mock.calls[0][0].toString()).toBe(JSON.stringify([]));
});
it("propagates compression failures without issuing a put", async () => {
@@ -199,7 +230,7 @@ describe("r2Storage.storeBatch", () => {
mocks.compress.mockRejectedValue(compressionError);
const storage = await loadStorage();
- await expect(storage.storeBatch(1, "session", [{}])).rejects.toBe(compressionError);
+ await expect(storage.storeBatch(1, "session", ["{}"])).rejects.toBe(compressionError);
expect(mocks.send).not.toHaveBeenCalled();
expect(console.error).toHaveBeenCalledWith("[R2Storage] Failed to store batch:", compressionError);
});
@@ -209,7 +240,7 @@ describe("r2Storage.storeBatch", () => {
mocks.send.mockRejectedValue(putError);
const storage = await loadStorage();
- await expect(storage.storeBatch(1, "session", [{}])).rejects.toBe(putError);
+ await expect(storage.storeBatch(1, "session", ["{}"])).rejects.toBe(putError);
expect(mocks.compress).toHaveBeenCalledOnce();
expect(console.error).toHaveBeenCalledWith("[R2Storage] Failed to store batch:", putError);
});
@@ -224,8 +255,8 @@ describe("r2Storage.storeBatch", () => {
mocks.send.mockResolvedValue({});
const storage = await loadStorage();
- const first = storage.storeBatch(1, "first", [{ sequence: 1 }]);
- const second = storage.storeBatch(2, "second", [{ sequence: 2 }]);
+ const first = storage.storeBatch(1, "first", [JSON.stringify({ sequence: 1 })]);
+ const second = storage.storeBatch(2, "second", [JSON.stringify({ sequence: 2 })]);
await expect(second).resolves.toBe("2/second/500.json.zst");
expect(mocks.send).toHaveBeenCalledOnce();
diff --git a/server/src/services/storage/r2StorageService.ts b/server/src/services/storage/r2StorageService.ts
index 921a0dc9e..198480524 100644
--- a/server/src/services/storage/r2StorageService.ts
+++ b/server/src/services/storage/r2StorageService.ts
@@ -67,8 +67,13 @@ class R2StorageService {
/**
* Store a batch of event data in R2
* Returns the storage key if successful, null if R2 is disabled
+ *
+ * Takes each event's data already serialized (`JSON.stringify(event.data)`),
+ * which ingestion computes anyway to size the batch. Joining those strings
+ * into an array literal yields the same bytes as `JSON.stringify` of the data
+ * array, without serializing the whole batch a second time.
*/
- async storeBatch(siteId: number, sessionId: string, eventDataArray: any[]): Promise<string | null> {
+ async storeBatch(siteId: number, sessionId: string, serializedEvents: string[]): Promise<string | null> {
if (!this.enabled || !this.client) {
return null;
}
@@ -78,7 +83,7 @@ class R2StorageService {
try {
// Compress with zstd - much faster decompression than brotli
- const jsonBuffer = Buffer.from(JSON.stringify(eventDataArray));
+ const jsonBuffer = Buffer.from(`[${serializedEvents.join(",")}]`);
const compressed = await zstdCompress(jsonBuffer, 3); // level 3 = good balance
await this.client.send(
@@ -91,7 +96,7 @@ class R2StorageService {
Metadata: {
siteId: siteId.toString(),
sessionId: sessionId,
- eventCount: eventDataArray.length.toString(),
+ eventCount: serializedEvents.length.toString(),
compression: "zstd",
},
})