:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { PipelineResultType, isDropResult } from '~/ingestion/framework/results'
import { createTestEventHeaders } from '~/tests/helpers/event-headers'
import { createTestPipelineEvent } from '~/tests/helpers/pipeline-event'
import { createTestTeam } from '~/tests/helpers/team'
import { EventHeaders, PipelineEvent, Team } from '~/types'
import { DropOldEventsInput, createDropOldEventsStep } from './drop-old-events-step'
describe('createDropOldEventsStep', () => {
const dropOldEventsStep = createDropOldEventsStep()
it.each([
{ dropThreshold: null, eventAgeSeconds: 7200, description: 'threshold is null' },
{ dropThreshold: 0, eventAgeSeconds: 7200, description: 'threshold is 0' },
{ dropThreshold: 3600, eventAgeSeconds: 1800, description: 'event is younger than threshold' },
{ dropThreshold: 3600, eventAgeSeconds: 3600, description: 'event is exactly at threshold' },
{ dropThreshold: 3600, eventAgeSeconds: -3600, description: 'event is in the future' },
])('passes through event when $description', async ({ dropThreshold, eventAgeSeconds }) => {
const input = createTestInput({ dropThreshold, eventAgeSeconds })
const result = await dropOldEventsStep(input)
expect(result.type).toBe(PipelineResultType.OK)
})
it('drops event older than threshold with warning', async () => {
const input = createTestInput({ dropThreshold: 3600, eventAgeSeconds: 7200 }) // threshold: 1h, age: 2h
const result = await dropOldEventsStep(input)
expect(result.type).toBe(PipelineResultType.DROP)
expect(isDropResult(result) && result.reason).toBe('event_too_old')
expect(result.warnings).toHaveLength(1)
expect(result.warnings[0]).toMatchObject({
type: 'event_dropped_too_old',
details: {
eventUuid: input.event.uuid,
event: '$pageview',
distinctId: 'user-1',
dropThresholdSeconds: 3600,
},
alwaysSend: false,
})
expect(result.warnings[0].details.ageInSeconds).toBeGreaterThanOrEqual(7199)
})
it('passes through when timestamp header is missing', async () => {
const input = createTestInput({
dropThreshold: 3600,
headers: { timestamp: undefined },
})
const result = await dropOldEventsStep(input)
expect(result.type).toBe(PipelineResultType.OK)
})
it('uses current time when now header is missing', async () => {
const input = createTestInput({
dropThreshold: 3600, // 1h threshold
eventAgeSeconds: 7200, // 2h old
headers: { now: undefined },
})
const result = await dropOldEventsStep(input)
// Event should be dropped based on timestamp vs current time
expect(result.type).toBe(PipelineResultType.DROP)
})
it('uses timestamp from headers (already normalized by Rust capture service)', async () => {
const now = DateTime.utc()
// Timestamp header is set to 2h ago by capture service (after clock skew correction)
const normalizedTimestamp = now.minus({ hours: 2 })
const input = createTestInput({
dropThreshold: 3600, // 1h threshold
event: {
// Event body timestamp is irrelevant - we use header
timestamp: now.minus({ minutes: 1 }).toISO()!,
},
headers: {
// Capture service already normalized this with clock skew correction
timestamp: normalizedTimestamp.toMillis().toString(),
now: now.toJSDate(),
},
})
const result = await dropOldEventsStep(input)
// Event should be dropped based on header timestamp (2h old), not event body (1m old)
expect(result.type).toBe(PipelineResultType.DROP)
})
describe('robustness to invalid data', () => {
it.each([
{ headers: undefined, description: 'headers is undefined' },
{ headers: null, description: 'headers is null' },
])('passes through when $description', async ({ headers }) => {
const input = createTestInput({
dropThreshold: 3600,
headers: headers as null | undefined,
})
const result = await dropOldEventsStep(input)
expect(result.type).toBe(PipelineResultType.OK)
})
it.each([
{ timestamp: 'not-a-number', description: 'timestamp is not a number' },
{ timestamp: '', description: 'timestamp is empty string' },
{ timestamp: 'NaN', description: 'timestamp parses to NaN' },
{ timestamp: 'Infinity', description: 'timestamp parses to Infinity' },
{ timestamp: '-Infinity', description: 'timestamp parses to -Infinity' },
])('passes through when $description', async ({ timestamp }) => {
const input = createTestInput({
dropThreshold: 3600,
headers: { timestamp },
})
const result = await dropOldEventsStep(input)
expect(result.type).toBe(PipelineResultType.OK)
})
it.each([
{ now: new Date('invalid'), description: 'now is invalid Date' },
{ now: 'not-a-date' as unknown as Date, description: 'now is a string' },
{ now: 12345 as unknown as Date, description: 'now is a number' },
{ now: {} as unknown as Date, description: 'now is an object' },
])('uses current time when $description', async ({ now }) => {
const currentTime = DateTime.utc()
const eventTimestamp = currentTime.minus({ hours: 2 })
const input = createTestInput({
dropThreshold: 3600,
headers: {
timestamp: eventTimestamp.toMillis().toString(),
now,
},
})
const result = await dropOldEventsStep(input)
// Should still drop because it falls back to current time
expect(result.type).toBe(PipelineResultType.DROP)
})
})
})
function createTestInput(options: {
dropThreshold?: number | null
eventAgeSeconds?: number
event?: Partial<PipelineEvent>
team?: Partial<Team>
headers?: Partial<EventHeaders> | null
}): DropOldEventsInput {
const { dropThreshold = null, eventAgeSeconds = 0, event = {}, team = {} } = options
const now = DateTime.utc()
const eventTimestamp = now.minus({ seconds: eventAgeSeconds })
// Only treat headers as null/undefined if explicitly passed as such
const headersExplicitlyNull = 'headers' in options && (options.headers === null || options.headers === undefined)
const resolvedHeaders = headersExplicitlyNull
? (options.headers as unknown as Pick<EventHeaders, 'timestamp' | 'now'>)
: createTestEventHeaders({
timestamp: eventTimestamp.toMillis().toString(),
now: now.toJSDate(),
...options.headers,
})
return {
event: createTestPipelineEvent({
event: '$pageview',
distinct_id: 'user-1',
ip: null,
site_url: 'https://test.posthog.com',
now: DateTime.utc().toISO()!,
...event,
}),
team: createTestTeam({ drop_events_older_than_seconds: dropThreshold, ...team }),
headers: resolvedHeaders,
}
}