:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { promisify } from 'node:util'
import { gzip, gzipSync } from 'zlib'
import { PipelineResultType } from '~/ingestion/framework/results'
import { SessionRecordingIngesterMetrics } from './metrics'
import {
ParseMessageStepInput,
createParseMessageStep,
decompressMessageValue,
observeClockSkew,
splitByDecodedBytes,
statedDecodedBytes,
} from './parse-message-step'
import { SessionReplayHeaders } from './pipeline-types'
const compressWithGzip = promisify(gzip)
describe('createParseMessageStep', () => {
// Use a fixed time for tests that need predictable timestamp behavior
const fixedTime = new Date('2023-01-01T00:00:00.000Z')
const fixedTimeMs = fixedTime.getTime() // 1672531200000
afterEach(() => {
jest.useRealTimers()
})
function createSnapshotPayload(options: {
sessionId: string
windowId?: string
snapshotItems: Array<{ type: number; timestamp: number }>
snapshotSource?: string
lib?: string
distinctId?: string
sentAt?: string
now?: string
}): string {
const event = {
event: '$snapshot_items',
properties: {
$session_id: options.sessionId,
$window_id: options.windowId ?? 'window-1',
$snapshot_items: options.snapshotItems,
...(options.snapshotSource !== undefined && { $snapshot_source: options.snapshotSource }),
...(options.lib !== undefined && { $lib: options.lib }),
},
}
const rawMessage: Record<string, unknown> = {
data: JSON.stringify(event),
}
if (options.distinctId !== undefined) {
rawMessage.distinct_id = options.distinctId
}
if (options.sentAt !== undefined) {
rawMessage.sent_at = options.sentAt
}
if (options.now !== undefined) {
rawMessage.now = options.now
}
return JSON.stringify(rawMessage)
}
function createValidSnapshotPayload(sessionId: string, windowId = 'window-1'): string {
const now = Date.now()
return createSnapshotPayload({
sessionId,
windowId,
snapshotItems: [
{ type: 2, timestamp: now },
{ type: 3, timestamp: now + 1000 },
],
distinctId: 'user-123',
})
}
function createMessage(
partition: number,
offset: number,
payload?: string,
headers?: Record<string, string>,
overrides?: Partial<Message>
): Message {
const kafkaHeaders: Message['headers'] = headers
? Object.entries(headers).map(([key, value]) => ({ [key]: Buffer.from(value) }))
: []
const message: Message = {
partition,
offset,
topic: 'test-topic',
value: payload ? Buffer.from(payload) : null,
key: Buffer.from('test-key'),
timestamp: Date.now(),
headers: kafkaHeaders,
size: payload?.length ?? 0,
}
return { ...message, ...overrides }
}
// The validate step guarantees these before parse; they must mirror the message body's
// session_id/distinct_id or parse DLQs the message as inconsistent.
function createReplayHeaders(overrides: Partial<SessionReplayHeaders> = {}): SessionReplayHeaders {
return { token: 'test-token', session_id: 'session-1', distinct_id: 'user-123', ...overrides }
}
function createInput(
partition: number,
offset: number,
payload?: string,
headers?: Record<string, string>,
overrides?: Partial<Message>,
replayHeaders: SessionReplayHeaders = createReplayHeaders()
): ParseMessageStepInput {
return {
message: createMessage(partition, offset, payload, headers, overrides),
headers: replayHeaders,
}
}
it('should parse valid messages and return ok results', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.session_id).toBe('session-1')
expect(result.value.parsedMessage.distinct_id).toBe('user-123')
}
})
it('should send messages with no value to DLQ', async () => {
const step = createParseMessageStep()
const input = createInput(0, 1, undefined)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('message_value_or_timestamp_is_empty')
}
})
it('should send messages with invalid JSON to DLQ', async () => {
const step = createParseMessageStep()
const input = createInput(0, 1, 'not valid json')
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('invalid_json')
}
})
it('should send messages with invalid message payload schema to DLQ', async () => {
const step = createParseMessageStep()
const input = createInput(0, 1, JSON.stringify({ invalid: 'schema' }))
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('invalid_message_payload')
}
})
it('should send non-snapshot messages to DLQ', async () => {
const step = createParseMessageStep()
const payload = JSON.stringify({
distinct_id: 'user-123',
data: JSON.stringify({ event: 'some_other_event', properties: {} }),
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('received_non_snapshot_message')
}
})
it('should drop messages with no valid rrweb events with ingestion warning', async () => {
const step = createParseMessageStep()
const event = {
event: '$snapshot_items',
properties: {
$session_id: 'session-1',
$window_id: 'window-1',
$snapshot_items: [], // Empty events array
},
}
const payload = JSON.stringify({
distinct_id: 'user-123',
data: JSON.stringify(event),
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DROP)
if (result.type === PipelineResultType.DROP) {
expect(result.reason).toBe('message_contained_no_valid_rrweb_events')
expect(result.warnings).toHaveLength(1)
expect(result.warnings[0]).toEqual({
type: 'message_contained_no_valid_rrweb_events',
details: {},
})
}
})
describe('message limits', () => {
const limits = { maxWindowIdBytes: 8, maxEventsPerMessage: 2 }
const event = { type: 2, timestamp: fixedTimeMs }
beforeEach(() => {
jest.useFakeTimers({ now: fixedTime })
})
afterEach(() => {
jest.restoreAllMocks()
})
it.each([
[
'the window ID',
'w'.repeat(9),
[event],
'message_window_id_too_long',
{ sessionId: 'session-1', windowIdBytes: 9, events: 1 },
],
[
'the window ID in bytes',
'é'.repeat(5),
[event],
'message_window_id_too_long',
{ sessionId: 'session-1', windowIdBytes: 10, events: 1 },
],
[
'the event count',
'w',
[event, event, event],
'message_too_many_events',
{ sessionId: 'session-1', windowIdBytes: 1, events: 3 },
],
] as const)(
'drops a message past the limit on %s when enforced',
async (_, windowId, items, reason, details) => {
const payload = createSnapshotPayload({
sessionId: 'session-1',
distinctId: 'user-123',
windowId,
snapshotItems: [...items],
})
const result = await createParseMessageStep({ ...limits, enforce: true })(createInput(0, 1, payload))
expect(result).toMatchObject({
type: PipelineResultType.DROP,
reason,
warnings: [{ type: reason, details }],
})
}
)
it.each([
['past a limit, not enforced', 'w'.repeat(9), false, ['message_window_id_too_long']],
['at the limits, enforced', 'w'.repeat(8), true, []],
])('keeps a message %s', async (_, windowId, enforce, counted) => {
const overLimitSpy = jest
.spyOn(SessionRecordingIngesterMetrics, 'incrementMessagesOverLimit')
.mockImplementation()
const payload = createSnapshotPayload({
sessionId: 'session-1',
distinctId: 'user-123',
windowId,
snapshotItems: [event, event],
})
const result = await createParseMessageStep({ ...limits, enforce })(createInput(0, 1, payload))
expect(result.type).toBe(PipelineResultType.OK)
expect(overLimitSpy.mock.calls.map(([limit]) => limit)).toEqual(counted)
})
})
describe('splitByDecodedBytes', () => {
const raw = (bytes: number): Message => ({ value: Buffer.alloc(bytes) }) as Message
const lz4Stating = (bytes: number): Message => {
const value = Buffer.alloc(8)
value.writeUInt32LE(bytes, 0)
return { value, headers: [{ 'content-encoding': 'lz4' }] } as unknown as Message
}
it.each([
['keeps one part when off', [raw(10), raw(10)], 0, [2]],
['keeps messages that fit together', [raw(10), raw(10)], 20, [2]],
['starts a new part past the limit', [raw(10), raw(10), raw(1)], 15, [1, 2]],
['gives a message over the limit its own part', [raw(1), raw(30), raw(1)], 10, [1, 1, 1]],
['counts the stated lz4 size', [lz4Stating(1000), lz4Stating(1000)], 1500, [1, 1]],
['counts the bytes when a stated size is past the cap', [lz4Stating(2 ** 31), raw(4)], 20, [2]],
] as const)('%s', (_, messages, maxBytes, sizes) => {
const parts = splitByDecodedBytes([...messages], maxBytes)
expect(parts.map((part) => part.length)).toEqual(sizes)
expect(parts.flat()).toEqual(messages)
})
it('counts the stated gzip size', () => {
const gzipped = gzipSync(Buffer.alloc(1000))
expect(statedDecodedBytes({ value: gzipped } as Message)).toBe(1000)
})
})
it('should handle gzipped messages', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-gzip')
const gzippedPayload = await compressWithGzip(Buffer.from(payload))
const message: Message = {
partition: 0,
offset: 1,
topic: 'test-topic',
value: gzippedPayload,
key: Buffer.from('test-key'),
timestamp: Date.now(),
headers: [],
size: gzippedPayload.length,
}
const result = await step({ message, headers: createReplayHeaders({ session_id: 'session-gzip' }) })
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.session_id).toBe('session-gzip')
}
})
it('should set token from the validated session replay headers', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(0, 1, payload, undefined, undefined, createReplayHeaders({ token: 'my-team-token' }))
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.token).toBe('my-team-token')
}
})
it('DLQs when the header session_id does not match the message body', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(0, 1, payload, undefined, undefined, createReplayHeaders({ session_id: 'session-2' }))
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('session_id_header_body_mismatch')
}
})
it('DLQs when the header distinct_id does not match the message body', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1') // body distinct_id is user-123
const input = createInput(
0,
1,
payload,
undefined,
undefined,
createReplayHeaders({ distinct_id: 'someone-else' })
)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('distinct_id_header_body_mismatch')
}
})
it('should populate metadata from message', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(5, 42, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.metadata.partition).toBe(5)
expect(result.value.parsedMessage.metadata.offset).toBe(42)
expect(result.value.parsedMessage.metadata.topic).toBe('test-topic')
}
})
it('should calculate events range from snapshot items', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.eventsRange.start.isValid).toBe(true)
expect(result.value.parsedMessage.eventsRange.end.isValid).toBe(true)
expect(result.value.parsedMessage.eventsRange.end >= result.value.parsedMessage.eventsRange.start).toBe(
true
)
}
})
it('should send messages with missing timestamp to DLQ', async () => {
const step = createParseMessageStep()
const payload = createValidSnapshotPayload('session-1')
const input = createInput(0, 1, payload, undefined, { timestamp: undefined })
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('message_value_or_timestamp_is_empty')
}
})
it('should send messages with invalid gzip data to DLQ', async () => {
const step = createParseMessageStep()
// Create a message with gzip magic bytes but invalid content
const message: Message = {
partition: 0,
offset: 1,
topic: 'test-topic',
value: Buffer.from([0x1f, 0x8b, 0x08, 0x00, 0x00]),
key: Buffer.from('test-key'),
timestamp: Date.now(),
headers: [],
size: 5,
}
const result = await step({ message, headers: createReplayHeaders() })
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('invalid_compressed_data')
}
})
it('should send messages with missing distinct_id to DLQ', async () => {
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [{ type: 2, timestamp: Date.now() }],
// distinctId intentionally omitted
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DLQ)
if (result.type === PipelineResultType.DLQ) {
expect(result.reason).toBe('invalid_message_payload')
}
})
it('should include snapshot_source and snapshot_library when present', async () => {
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 2, timestamp: Date.now() },
{ type: 3, timestamp: Date.now() + 1000 },
],
snapshotSource: 'test-source',
lib: 'test-lib',
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.snapshot_source).toBe('test-source')
expect(result.value.parsedMessage.snapshot_library).toBe('test-lib')
}
})
it('should set snapshot_source and snapshot_library to null when not present', async () => {
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 2, timestamp: Date.now() },
{ type: 3, timestamp: Date.now() + 1000 },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.snapshot_source).toBeNull()
expect(result.value.parsedMessage.snapshot_library).toBeNull()
}
})
it('should filter out events with zero or negative timestamps', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: fixedTimeMs },
{ type: 2, timestamp: 0 },
{ type: 3, timestamp: -1000 },
{ type: 4, timestamp: fixedTimeMs + 1000 },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
const events = result.value.parsedMessage.eventsByWindowId['window-1']
expect(events).toHaveLength(2)
expect(events).toEqual([
{ type: 1, timestamp: fixedTimeMs },
{ type: 4, timestamp: fixedTimeMs + 1000 },
])
}
})
it('should use min/max for timestamp range instead of first/last', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: fixedTimeMs + 2000 }, // Not the smallest
{ type: 2, timestamp: fixedTimeMs }, // Smallest
{ type: 3, timestamp: fixedTimeMs + 4000 }, // Not the largest
{ type: 4, timestamp: fixedTimeMs + 5000 }, // Largest
{ type: 5, timestamp: fixedTimeMs + 3000 }, // Middle
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.eventsRange.start.toMillis()).toBe(fixedTimeMs)
expect(result.value.parsedMessage.eventsRange.end.toMillis()).toBe(fixedTimeMs + 5000)
}
})
it('should drop messages with events too far in the future (>7 days)', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const eightDaysInFuture = fixedTimeMs + 8 * 24 * 60 * 60 * 1000
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: fixedTimeMs },
{ type: 2, timestamp: eightDaysInFuture },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DROP)
if (result.type === PipelineResultType.DROP) {
expect(result.reason).toBe('message_timestamp_diff_too_large')
}
})
it('should drop messages with events too far in the past (>7 days)', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const eightDaysInPast = fixedTimeMs - 8 * 24 * 60 * 60 * 1000
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: fixedTimeMs },
{ type: 2, timestamp: eightDaysInPast },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DROP)
if (result.type === PipelineResultType.DROP) {
expect(result.reason).toBe('message_timestamp_diff_too_large')
}
})
it('should accept messages with events within 7 days', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const sixDaysInFuture = fixedTimeMs + 6 * 24 * 60 * 60 * 1000
const sixDaysInPast = fixedTimeMs - 6 * 24 * 60 * 60 * 1000
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: sixDaysInPast },
{ type: 2, timestamp: fixedTimeMs },
{ type: 3, timestamp: sixDaysInFuture },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.eventsByWindowId['window-1']).toHaveLength(3)
expect(result.value.parsedMessage.eventsRange.start.toMillis()).toBe(sixDaysInPast)
expect(result.value.parsedMessage.eventsRange.end.toMillis()).toBe(sixDaysInFuture)
}
})
it('should include ingestion warning when dropping messages with timestamp diff too large', async () => {
jest.useFakeTimers()
jest.setSystemTime(fixedTime)
const step = createParseMessageStep()
const eightDaysInFuture = fixedTimeMs + 8 * 24 * 60 * 60 * 1000
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 1, timestamp: fixedTimeMs },
{ type: 2, timestamp: eightDaysInFuture },
],
distinctId: 'user-123',
})
const input = createInput(0, 1, payload)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.DROP)
expect(result.warnings).toHaveLength(1)
expect(result.warnings[0]).toEqual({
type: 'message_timestamp_diff_too_large',
details: {
startDiffDays: expect.any(Number),
endDiffDays: expect.any(Number),
thresholdDays: 7,
},
})
// The end timestamp is 8 days in the future
expect(result.warnings[0].details.endDiffDays).toBeGreaterThanOrEqual(8)
})
it.each([
['uppercase UUID', '0192E72A-1DD2-7714-8000-8B3E4C123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
['lowercase UUID', '0192e72a-1dd2-7714-8000-8b3e4c123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
['mixed case UUID', '0192E72a-1Dd2-7714-8000-8b3E4c123456', '0192e72a-1dd2-7714-8000-8b3e4c123456'],
['non-UUID session ID', 'custom-session-id', 'custom-session-id'],
])('should handle %s session_id: %s -> %s', async (_desc, inputSessionId, expectedSessionId) => {
const step = createParseMessageStep()
const payload = createSnapshotPayload({
sessionId: inputSessionId,
snapshotItems: [
{ type: 2, timestamp: Date.now() },
{ type: 3, timestamp: Date.now() + 1000 },
],
distinctId: 'user-123',
})
// The header id is already normalized by the validate step; the body carries the raw id.
const input = createInput(
0,
1,
payload,
undefined,
undefined,
createReplayHeaders({ session_id: expectedSessionId })
)
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
expect(result.value.parsedMessage.session_id).toBe(expectedSessionId)
}
})
describe('clock skew', () => {
const HOUR_MS = 60 * 60 * 1000
const serverNow = '2023-01-01T00:00:00.000Z'
let observeSpy: jest.SpyInstance
let unmeasuredSpy: jest.SpyInstance
beforeEach(() => {
observeSpy = jest.spyOn(SessionRecordingIngesterMetrics, 'observeMessageClockSkew').mockImplementation()
unmeasuredSpy = jest
.spyOn(SessionRecordingIngesterMetrics, 'incrementMessageClockSkewUnmeasured')
.mockImplementation()
})
afterEach(() => {
jest.restoreAllMocks()
})
it.each([
['reads 1h ahead of the server', '2023-01-01T01:00:00.000Z', 'device_ahead', 3600],
['reads 1h behind the server', '2022-12-31T23:00:00.000Z', 'device_behind', 3600],
['agrees with the server', serverNow, 'device_ahead', 0],
])('records the offset when the device clock %s', (_desc, sentAt, direction, seconds) => {
observeClockSkew(sentAt, serverNow)
expect(observeSpy).toHaveBeenCalledWith(direction, seconds)
expect(unmeasuredSpy).not.toHaveBeenCalled()
})
it.each([
['capture recorded no sent_at', undefined, serverNow],
['sent_at is unparseable', 'not-a-date', serverNow],
['capture recorded no now', '2023-01-01T01:00:00.000Z', undefined],
])('counts the message as unmeasured when %s', (_desc, sentAt, now) => {
observeClockSkew(sentAt, now)
expect(observeSpy).not.toHaveBeenCalled()
expect(unmeasuredSpy).toHaveBeenCalledTimes(1)
})
it('reports no offset for a buffered upload from a correctly clocked device', async () => {
const step = createParseMessageStep()
const now = Date.now()
const uploadedAt = new Date(now).toISOString()
const payload = createSnapshotPayload({
sessionId: 'session-1',
// Buffering, not a wrong clock: the Kafka message time would read this as an hour of skew.
snapshotItems: [{ type: 2, timestamp: now - HOUR_MS }],
distinctId: 'user-123',
sentAt: uploadedAt,
now: uploadedAt,
})
const input = createInput(0, 1, payload, undefined, { timestamp: now })
const result = await step(input)
expect(result.type).toBe(PipelineResultType.OK)
expect(result.warnings).toHaveLength(0)
expect(observeSpy).toHaveBeenCalledWith('device_ahead', 0)
})
})
describe('clock skew correction', () => {
const SEVEN_HOURS_MS = 7 * 60 * 60 * 1000
function skewedInput(options: {
baseMs: number
deviceOffsetMs?: number
latencyMs?: number
sentAt?: string | null
now?: string | null
}): ParseMessageStepInput {
const { baseMs, deviceOffsetMs = 0, latencyMs = 0 } = options
const now = options.now === undefined ? new Date(baseMs + latencyMs).toISOString() : options.now
const sentAt =
options.sentAt === undefined ? new Date(baseMs + deviceOffsetMs).toISOString() : options.sentAt
const payload = createSnapshotPayload({
sessionId: 'session-1',
snapshotItems: [
{ type: 2, timestamp: baseMs + deviceOffsetMs },
{ type: 3, timestamp: baseMs + deviceOffsetMs + 1000 },
],
distinctId: 'user-123',
...(now !== null && { now }),
...(sentAt !== null && { sentAt }),
})
return createInput(0, 1, payload)
}
async function startMsOf(input: ParseMessageStepInput): Promise<number> {
const result = await createParseMessageStep()(input)
expect(result.type).toBe(PipelineResultType.OK)
if (result.type !== PipelineResultType.OK) {
throw new Error('expected an ok result')
}
return result.value.parsedMessage.eventsRange.start.toMillis()
}
it.each([
['ahead', SEVEN_HOURS_MS],
['behind', -SEVEN_HOURS_MS],
])('shifts rrweb timestamps to server time when the device clock is %s', async (_name, deviceOffsetMs) => {
const baseMs = Date.now()
const step = createParseMessageStep()
const result = await step(skewedInput({ baseMs, deviceOffsetMs }))
expect(result.type).toBe(PipelineResultType.OK)
if (result.type === PipelineResultType.OK) {
const { parsedMessage } = result.value
expect(parsedMessage.eventsRange.start.toMillis()).toBe(baseMs)
expect(parsedMessage.eventsRange.end.toMillis()).toBe(baseMs + 1000)
expect(parsedMessage.eventsByWindowId['window-1'].map((e) => e.timestamp)).toEqual([
baseMs,
baseMs + 1000,
])
}
})
it('corrects two messages of one session identically despite different latencies', async () => {
const baseMs = Date.now()
const fast = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, latencyMs: 50 }))
const slow = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, latencyMs: 2500 }))
expect(fast).toBe(slow)
})
it('leaves a well-behaved clock untouched, so latency cannot reorder its events', async () => {
const baseMs = Date.now()
const startMs = await startMsOf(skewedInput({ baseMs, deviceOffsetMs: 400, latencyMs: 900 }))
expect(startMs).toBe(baseMs + 400)
})
const validIso = '2023-01-01T00:00:00.000Z'
it.each([
['sent_at is missing', { sentAt: null }],
['now is missing', { now: null }],
['sent_at is unparseable', { sentAt: 'not-a-date' }],
['now is unparseable', { now: 'not-a-date' }],
])(
'leaves timestamps unchanged when %s',
async (_name, overrides: { now?: string | null; sentAt?: string | null }) => {
const baseMs = Date.now()
const startMs = await startMsOf(
skewedInput({ baseMs, deviceOffsetMs: SEVEN_HOURS_MS, now: validIso, ...overrides })
)
expect(startMs).toBe(baseMs + SEVEN_HOURS_MS)
}
)
})
describe('decompressMessageValue lz4 hardening', () => {
const lz4 = require('lz4')
function lz4Message(value: Buffer): Message {
return {
value,
headers: [{ 'content-encoding': Buffer.from('lz4') }],
} as unknown as Message
}
function lz4Frame(payload: Buffer, claimedSize?: number): Buffer {
const compressed = Buffer.alloc(lz4.encodeBound(payload.length))
const n = lz4.encodeBlock(payload, compressed)
const prefix = Buffer.alloc(4)
prefix.writeUInt32LE(claimedSize ?? payload.length, 0)
return Buffer.concat([prefix, compressed.subarray(0, n)])
}
it('round-trips a valid block', () => {
const payload = Buffer.from('{"distinct_id":"d","data":"x"}'.repeat(10))
expect(decompressMessageValue(lz4Message(lz4Frame(payload)))).toEqual(payload)
})
it('rejects a size prefix beyond the decompression cap instead of allocating it', () => {
const payload = Buffer.from('small')
const bomb = lz4Frame(payload, 0xffffffff)
expect(() => decompressMessageValue(lz4Message(bomb))).toThrow(/exceeds/)
})
it('rejects a size prefix that does not match the decoded length', () => {
const payload = Buffer.from('0123456789'.repeat(20))
const lying = lz4Frame(payload, payload.length + 512)
expect(() => decompressMessageValue(lz4Message(lying))).toThrow(/decoded/)
})
})
})